{"id":"https://openalex.org/W4401408735","doi":"https://doi.org/10.1145/3673038.3673095","title":"Holmes: Towards Distributed Training Across Clusters with Heterogeneous NIC Environment","display_name":"Holmes: Towards Distributed Training Across Clusters with Heterogeneous NIC Environment","publication_year":2024,"publication_date":"2024-08-08","ids":{"openalex":"https://openalex.org/W4401408735","doi":"https://doi.org/10.1145/3673038.3673095"},"language":"en","primary_location":{"id":"doi:10.1145/3673038.3673095","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673038.3673095","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673038.3673095","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 53rd International Conference on Parallel Processing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3673038.3673095","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065197504","display_name":"Fei Yang","orcid":"https://orcid.org/0000-0003-4802-3191"},"institutions":[{"id":"https://openalex.org/I4210123185","display_name":"Zhejiang Lab","ror":"https://ror.org/02m2h7991","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210123185"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fei Yang","raw_affiliation_strings":["Zhejiang Lab, China"],"raw_orcid":"https://orcid.org/0000-0003-4802-3191","affiliations":[{"raw_affiliation_string":"Zhejiang Lab, China","institution_ids":["https://openalex.org/I4210123185"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012097021","display_name":"Shuang Peng","orcid":null},"institutions":[{"id":"https://openalex.org/I4210123185","display_name":"Zhejiang Lab","ror":"https://ror.org/02m2h7991","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210123185"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuang Peng","raw_affiliation_strings":["Zhejiang Lab, China"],"raw_orcid":"https://orcid.org/0009-0001-9599-4276","affiliations":[{"raw_affiliation_string":"Zhejiang Lab, China","institution_ids":["https://openalex.org/I4210123185"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ning Sun","orcid":"https://orcid.org/0009-0002-7859-4454"},"institutions":[{"id":"https://openalex.org/I4210123185","display_name":"Zhejiang Lab","ror":"https://ror.org/02m2h7991","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210123185"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ning Sun","raw_affiliation_strings":["Zhejiang Lab, China"],"raw_orcid":"https://orcid.org/0009-0002-7859-4454","affiliations":[{"raw_affiliation_string":"Zhejiang Lab, China","institution_ids":["https://openalex.org/I4210123185"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Fangyu Wang","orcid":"https://orcid.org/0009-0002-3748-5606"},"institutions":[{"id":"https://openalex.org/I4210123185","display_name":"Zhejiang Lab","ror":"https://ror.org/02m2h7991","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210123185"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fangyu Wang","raw_affiliation_strings":["Zhejiang Lab, China"],"raw_orcid":"https://orcid.org/0009-0002-3748-5606","affiliations":[{"raw_affiliation_string":"Zhejiang Lab, China","institution_ids":["https://openalex.org/I4210123185"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111283890","display_name":"Yuanyuan Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210123185","display_name":"Zhejiang Lab","ror":"https://ror.org/02m2h7991","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210123185"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuanyuan Wang","raw_affiliation_strings":["Zhejiang Lab, China"],"raw_orcid":"https://orcid.org/0009-0007-3205-1442","affiliations":[{"raw_affiliation_string":"Zhejiang Lab, China","institution_ids":["https://openalex.org/I4210123185"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100868455","display_name":"Fu Zhong Wu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210123185","display_name":"Zhejiang Lab","ror":"https://ror.org/02m2h7991","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210123185"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fu Wu","raw_affiliation_strings":["Zhejiang Lab, China"],"raw_orcid":"https://orcid.org/0009-0001-9613-5908","affiliations":[{"raw_affiliation_string":"Zhejiang Lab, China","institution_ids":["https://openalex.org/I4210123185"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083333630","display_name":"Jiezhong Qiu","orcid":"https://orcid.org/0000-0001-9514-0708"},"institutions":[{"id":"https://openalex.org/I4210123185","display_name":"Zhejiang Lab","ror":"https://ror.org/02m2h7991","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210123185"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiezhong Qiu","raw_affiliation_strings":["Zhejiang Lab, China"],"raw_orcid":"https://orcid.org/0000-0001-9514-0708","affiliations":[{"raw_affiliation_string":"Zhejiang Lab, China","institution_ids":["https://openalex.org/I4210123185"]}]},{"author_position":"last","author":{"id":null,"display_name":"Aimin Pan","orcid":"https://orcid.org/0009-0001-5996-9110"},"institutions":[{"id":"https://openalex.org/I4210123185","display_name":"Zhejiang Lab","ror":"https://ror.org/02m2h7991","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210123185"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Aimin Pan","raw_affiliation_strings":["Zhejiang Lab, China"],"raw_orcid":"https://orcid.org/0009-0001-5996-9110","affiliations":[{"raw_affiliation_string":"Zhejiang Lab, China","institution_ids":["https://openalex.org/I4210123185"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210123185"],"apc_list":null,"apc_paid":null,"fwci":2.7925,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.90940794,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"514","last_page":"523"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9919000267982483,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6618067622184753},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5515857338905334},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.39681389927864075},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.08777734637260437}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6618067622184753},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5515857338905334},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.39681389927864075},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.08777734637260437},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3673038.3673095","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673038.3673095","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673038.3673095","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 53rd International Conference on Parallel Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3673038.3673095","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673038.3673095","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673038.3673095","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 53rd International Conference on Parallel Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.41999998688697815,"display_name":"Climate action","id":"https://metadata.un.org/sdg/13"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4401408735.pdf"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W2004772832","https://openalex.org/W2045271686","https://openalex.org/W2057332538","https://openalex.org/W2060393849","https://openalex.org/W2119391823","https://openalex.org/W2338370098","https://openalex.org/W2498764059","https://openalex.org/W2612387305","https://openalex.org/W2734941459","https://openalex.org/W2887718373","https://openalex.org/W2901299405","https://openalex.org/W2926767350","https://openalex.org/W2969388332","https://openalex.org/W3081168214","https://openalex.org/W3101708369","https://openalex.org/W3129831491","https://openalex.org/W3204998121","https://openalex.org/W4281728635","https://openalex.org/W6778883912","https://openalex.org/W6799065049"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W230091440","https://openalex.org/W2390279801","https://openalex.org/W2233261550","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2376932109"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,23],"(LLMs)":[3],"such":[4,68],"as":[5],"GPT-3,":[6],"OPT,":[7],"and":[8,65,79,106,234],"LLaMA":[9],"have":[10],"demonstrated":[11],"remarkable":[12],"accuracy":[13],"in":[14,46,120,131,163,170,181,229],"a":[15,96,121],"wide":[16],"range":[17],"of":[18,31,33,37,67,137,144,231],"tasks.":[19],"However,":[20],"training":[21,42,75,85,97,133,207,232],"these":[22],"can":[24,235],"incur":[25],"significant":[26],"expenses,":[27],"often":[28],"requiring":[29],"tens":[30],"thousands":[32],"GPUs":[34],"for":[35,99],"months":[36],"continuous":[38],"operation.":[39],"Typically,":[40],"this":[41,91],"is":[43,71],"carried":[44],"out":[45],"specialized":[47],"GPU":[48,138,160],"clusters":[49,70],"equipped":[50],"with":[51,198,239],"homogeneous":[52,87,199],"high-speed":[53,166],"Remote":[54],"Direct":[55],"Memory":[56],"Access":[57],"(RDMA)":[58],"network":[59],"interface":[60],"cards":[61],"(NICs).":[62],"The":[63],"acquisition":[64],"maintenance":[66],"dedicated":[69],"challenging.":[72],"Current":[73],"LLM":[74,132,223],"frameworks,":[76],"like":[77],"Megatron-LM":[78],"Megatron-DeepSpeed,":[80],"focus":[81],"primarily":[82],"on":[83,141],"optimizing":[84],"within":[86,209],"cluster":[88],"settings.":[89],"In":[90,186],"paper,":[92],"we":[93,215],"introduce":[94],"Holmes,":[95],"framework":[98,190,219],"LLMs":[100],"that":[101,125,177,217],"employs":[102],"thoughtfully":[103],"crafted":[104],"data":[105],"model":[107],"parallelism":[108],"strategies":[109],"over":[110],"the":[111,142,182,210],"heterogeneous":[112,183,226],"NIC":[113,184,227],"environment.":[114,185,213],"Our":[115],"primary":[116],"technical":[117],"contribution":[118],"lies":[119],"novel":[122],"scheduling":[123],"method":[124],"intelligently":[126],"allocates":[127],"distinct":[128,171],"computational":[129],"tasklets":[130],"to":[134,158,195],"specific":[135],"groups":[136],"devices":[139],"based":[140],"characteristics":[143],"their":[145],"connected":[146],"NICs.":[147],"Furthermore,":[148],"our":[149,189,218],"proposed":[150],"framework,":[151],"utilizing":[152],"pipeline":[153],"parallel":[154],"techniques,":[155],"demonstrates":[156],"scalability":[157],"multiple":[159],"clusters,":[161],"even":[162],"scenarios":[164,180],"without":[165],"interconnects":[167],"between":[168],"nodes":[169],"clusters.":[172],"We":[173],"conducted":[174],"comprehensive":[175],"experiments":[176],"involved":[178],"various":[179],"most":[187],"cases,":[188],"achieves":[191],"performance":[192],"levels":[193],"close":[194],"those":[196],"achievable":[197],"RDMA-capable":[200],"networks":[201],"(InfiniBand":[202],"or":[203],"RoCE),":[204],"significantly":[205],"exceeding":[206],"efficiency":[208,233],"pure":[211],"Ethernet":[212],"Additionally,":[214],"verified":[216],"outperforms":[220],"other":[221],"mainstream":[222],"frameworks":[224],"under":[225],"environment":[228],"terms":[230],"be":[236],"seamlessly":[237],"integrated":[238],"them.":[240]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}