{"id":"https://openalex.org/W4402897082","doi":"https://doi.org/10.1109/iwqos61813.2024.10682856","title":"Lins: Reducing Communication Overhead of ZeRO for Efficient LLM Training","display_name":"Lins: Reducing Communication Overhead of ZeRO for Efficient LLM Training","publication_year":2024,"publication_date":"2024-06-19","ids":{"openalex":"https://openalex.org/W4402897082","doi":"https://doi.org/10.1109/iwqos61813.2024.10682856"},"language":"en","primary_location":{"id":"doi:10.1109/iwqos61813.2024.10682856","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iwqos61813.2024.10682856","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE/ACM 32nd International Symposium on Quality of Service (IWQoS)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068990415","display_name":"Qiaoling Chen","orcid":"https://orcid.org/0000-0002-2884-0824"},"institutions":[{"id":"https://openalex.org/I4210123532","display_name":"Nordic Transport Development","ror":"https://ror.org/0295ktr04","country_code":"DK","type":"company","lineage":["https://openalex.org/I4210123532"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Qiaoling Chen","raw_affiliation_strings":["NTU,S-Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NTU,S-Lab","institution_ids":["https://openalex.org/I4210123532"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035921543","display_name":"Qinghao Hu","orcid":"https://orcid.org/0000-0003-1034-7502"},"institutions":[{"id":"https://openalex.org/I4210123532","display_name":"Nordic Transport Development","ror":"https://ror.org/0295ktr04","country_code":"DK","type":"company","lineage":["https://openalex.org/I4210123532"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Qinghao Hu","raw_affiliation_strings":["NTU,S-Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NTU,S-Lab","institution_ids":["https://openalex.org/I4210123532"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019583668","display_name":"Guoteng Wang","orcid":"https://orcid.org/0000-0002-1018-5673"},"institutions":[{"id":"https://openalex.org/I4210122302","display_name":"ShangHai JiAi Genetics & IVF Institute","ror":"https://ror.org/02rgbry52","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210122302"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guoteng Wang","raw_affiliation_strings":["Shanghai AI Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory","institution_ids":["https://openalex.org/I4210122302","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085650484","display_name":"Yingtong Xiong","orcid":null},"institutions":[{"id":"https://openalex.org/I4210122302","display_name":"ShangHai JiAi Genetics & IVF Institute","ror":"https://ror.org/02rgbry52","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210122302"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yingtong Xiong","raw_affiliation_strings":["Shanghai AI Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory","institution_ids":["https://openalex.org/I4210122302","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015399692","display_name":"Ting Huang","orcid":"https://orcid.org/0000-0002-8669-4028"},"institutions":[{"id":"https://openalex.org/I4210128910","display_name":"Group Sense (China)","ror":"https://ror.org/036wd5777","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210128910"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ting Huang","raw_affiliation_strings":["SenseTime"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SenseTime","institution_ids":["https://openalex.org/I4210128910"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100451602","display_name":"Xun Chen","orcid":"https://orcid.org/0000-0002-4922-8116"},"institutions":[{"id":"https://openalex.org/I4210128910","display_name":"Group Sense (China)","ror":"https://ror.org/036wd5777","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210128910"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xun Chen","raw_affiliation_strings":["SenseTime"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SenseTime","institution_ids":["https://openalex.org/I4210128910"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013420855","display_name":"Yang Gao","orcid":"https://orcid.org/0000-0001-9328-1611"},"institutions":[{"id":"https://openalex.org/I4210122302","display_name":"ShangHai JiAi Genetics & IVF Institute","ror":"https://ror.org/02rgbry52","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210122302"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yang Gao","raw_affiliation_strings":["Shanghai AI Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory","institution_ids":["https://openalex.org/I4210122302","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055394194","display_name":"Hang Yan","orcid":"https://orcid.org/0000-0003-3912-0168"},"institutions":[{"id":"https://openalex.org/I4210122302","display_name":"ShangHai JiAi Genetics & IVF Institute","ror":"https://ror.org/02rgbry52","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210122302"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hang Yan","raw_affiliation_strings":["Shanghai AI Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory","institution_ids":["https://openalex.org/I4210122302","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041572550","display_name":"Yonggang Wen","orcid":"https://orcid.org/0000-0002-2751-5114"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yonggang Wen","raw_affiliation_strings":["Nanyang Technological University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanyang Technological University","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101591101","display_name":"Tianwei Zhang","orcid":"https://orcid.org/0000-0001-6595-6650"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Tianwei Zhang","raw_affiliation_strings":["Nanyang Technological University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanyang Technological University","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100578819","display_name":"Peng Sun","orcid":"https://orcid.org/0009-0003-2900-075X"},"institutions":[{"id":"https://openalex.org/I4210122302","display_name":"ShangHai JiAi Genetics & IVF Institute","ror":"https://ror.org/02rgbry52","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210122302"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peng Sun","raw_affiliation_strings":["Shanghai AI Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory","institution_ids":["https://openalex.org/I4210122302","https://openalex.org/I4391012619"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13344","display_name":"Industrial Automation and Control Systems","score":0.7652000188827515,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13344","display_name":"Industrial Automation and Control Systems","score":0.7652000188827515,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12203","display_name":"Mobile Agent-Based Network Management","score":0.7084000110626221,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10651","display_name":"IPv6, Mobility, Handover, Networks, Security","score":0.6370999813079834,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.7440277338027954},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.7345362901687622},{"id":"https://openalex.org/keywords/zero","display_name":"Zero (linguistics)","score":0.6607826948165894},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.654901385307312},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.10413694381713867},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.06807413697242737}],"concepts":[{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.7440277338027954},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.7345362901687622},{"id":"https://openalex.org/C2780813799","wikidata":"https://www.wikidata.org/wiki/Q3274237","display_name":"Zero (linguistics)","level":2,"score":0.6607826948165894},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.654901385307312},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.10413694381713867},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.06807413697242737},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/iwqos61813.2024.10682856","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iwqos61813.2024.10682856","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE/ACM 32nd International Symposium on Quality of Service (IWQoS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":43,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1558115001","https://openalex.org/W2160054705","https://openalex.org/W2577427573","https://openalex.org/W2969388332","https://openalex.org/W2975712713","https://openalex.org/W2991040477","https://openalex.org/W2994144272","https://openalex.org/W3019389960","https://openalex.org/W3081168214","https://openalex.org/W3086105743","https://openalex.org/W3129831491","https://openalex.org/W3132107458","https://openalex.org/W4221150592","https://openalex.org/W4224308101","https://openalex.org/W4225591000","https://openalex.org/W4229701516","https://openalex.org/W4292779060","https://openalex.org/W4312983671","https://openalex.org/W4322718191","https://openalex.org/W4327911839","https://openalex.org/W4372267133","https://openalex.org/W4381551031","https://openalex.org/W4384648639","https://openalex.org/W4385245566","https://openalex.org/W4386709668","https://openalex.org/W4386768656","https://openalex.org/W4387544258","https://openalex.org/W4401352466","https://openalex.org/W6631190155","https://openalex.org/W6739901393","https://openalex.org/W6753584990","https://openalex.org/W6756718674","https://openalex.org/W6778883912","https://openalex.org/W6799372109","https://openalex.org/W6810081322","https://openalex.org/W6810220367","https://openalex.org/W6810764171","https://openalex.org/W6811928498","https://openalex.org/W6850625674","https://openalex.org/W6853287796","https://openalex.org/W6854308872","https://openalex.org/W6862033441"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W230091440","https://openalex.org/W2390279801","https://openalex.org/W2233261550","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2376932109"],"abstract_inverted_index":{"Training":[0],"large":[1],"language":[2],"models":[3],"(LLMs)":[4],"encounters":[5],"challenges":[6,36],"in":[7,126,131],"GPU":[8],"memory":[9,15],"consumption":[10],"due":[11],"to":[12,49,78,102,111,135],"the":[13,71,87,104,119],"high":[14],"requirements":[16],"of":[17,95],"model":[18,72,121],"states.":[19],"The":[20],"widely":[21],"used":[22],"Zero":[23],"Redundancy":[24],"Optimizer":[25,76],"(ZeRO)":[26],"addresses":[27],"this":[28,41],"issue":[29],"through":[30],"strategic":[31],"sharding":[32,60,82,106],"but":[33],"introduces":[34],"communication":[35,96],"at":[37],"scale.":[38],"To":[39],"tackle":[40],"problem,":[42],"we":[43],"propose":[44],"Lins,":[45],"a":[46,81,92,127],"system":[47],"designed":[48],"optimize":[50],"ZeRO":[51],"for":[52],"scalable":[53],"LLM":[54],"training.":[55],"Lins":[56],"incorporates":[57],"three":[58],"flexible":[59],"strategies:":[61],"Full-Replica,":[62],"Full-Sharding,":[63],"and":[64,66,141],"Partial-Sharding,":[65],"allows":[67],"each":[68],"component":[69],"within":[70],"states":[73],"(Parameters,":[74],"Gradients,":[75],"States)":[77],"independently":[79],"choose":[80],"strategy":[83],"as":[84,86],"well":[85],"device":[88],"mesh.":[89],"We":[90],"conduct":[91],"thorough":[93],"analysis":[94],"costs,":[97],"formulating":[98],"an":[99],"optimization":[100],"problem":[101],"discover":[103],"optimal":[105],"strategy.":[107],"Evaluations":[108],"demonstrate":[109],"up":[110],"52%":[112],"Model":[113],"FLOPs":[114],"Utilization":[115],"(MFU)":[116],"when":[117],"training":[118,132],"LLaMA-based":[120],"on":[122],"1024":[123],"GPUs,":[124],"resulting":[125],"1.56":[128],"times":[129],"improvement":[130],"throughput":[133],"compared":[134],"newly":[136],"proposed":[137],"systems":[138],"like":[139],"MiCS":[140],"ZeRO++.":[142]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":4}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}