{"id":"https://openalex.org/W7124197254","doi":"https://doi.org/10.1109/icpads67057.2025.11323171","title":"Fine-Grained State Sharding and Communication Pipelining for Partially Sharded Data Parallelism","display_name":"Fine-Grained State Sharding and Communication Pipelining for Partially Sharded Data Parallelism","publication_year":2025,"publication_date":"2025-12-14","ids":{"openalex":"https://openalex.org/W7124197254","doi":"https://doi.org/10.1109/icpads67057.2025.11323171"},"language":null,"primary_location":{"id":"doi:10.1109/icpads67057.2025.11323171","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icpads67057.2025.11323171","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 31th International Conference on Parallel and Distributed Systems (ICPADS)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109715935","display_name":"Weimin Li","orcid":null},"institutions":[{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weimin Li","raw_affiliation_strings":["SKLP, Institute of Computing Technology, CAS,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SKLP, Institute of Computing Technology, CAS,Beijing,China","institution_ids":["https://openalex.org/I4210090176"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yuzhong Sun","orcid":null},"institutions":[{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuzhong Sun","raw_affiliation_strings":["SKLP, Institute of Computing Technology, CAS,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SKLP, Institute of Computing Technology, CAS,Beijing,China","institution_ids":["https://openalex.org/I4210090176"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078249204","display_name":"Meng Lin","orcid":"https://orcid.org/0000-0001-8920-5464"},"institutions":[{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lin Meng","raw_affiliation_strings":["SKLP, Institute of Computing Technology, CAS,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SKLP, Institute of Computing Technology, CAS,Beijing,China","institution_ids":["https://openalex.org/I4210090176"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210090176"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3409000039100647,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3409000039100647,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.2720000147819519,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.08219999819993973,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7383000254631042},{"id":"https://openalex.org/keywords/bandwidth","display_name":"Bandwidth (computing)","score":0.4778999984264374},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.44699999690055847},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.43849998712539673},{"id":"https://openalex.org/keywords/fusion","display_name":"Fusion","score":0.40389999747276306},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.3481999933719635},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.3294000029563904},{"id":"https://openalex.org/keywords/memory-footprint","display_name":"Memory footprint","score":0.32199999690055847}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8137000203132629},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7383000254631042},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5041999816894531},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.48240000009536743},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.4778999984264374},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.44699999690055847},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.43849998712539673},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.40389999747276306},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.3481999933719635},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.3294000029563904},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.3246999979019165},{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.32199999690055847},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.3001999855041504},{"id":"https://openalex.org/C158156997","wikidata":"https://www.wikidata.org/wiki/Q1416645","display_name":"Models of communication","level":2,"score":0.2948000133037567},{"id":"https://openalex.org/C106515295","wikidata":"https://www.wikidata.org/wiki/Q26806595","display_name":"Parallel processing","level":2,"score":0.2935999929904938},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.2809999883174896},{"id":"https://openalex.org/C2989134064","wikidata":"https://www.wikidata.org/wiki/Q288510","display_name":"Execution time","level":2,"score":0.2720000147819519},{"id":"https://openalex.org/C188045654","wikidata":"https://www.wikidata.org/wiki/Q17148339","display_name":"Memory bandwidth","level":2,"score":0.2678999900817871},{"id":"https://openalex.org/C2776834041","wikidata":"https://www.wikidata.org/wiki/Q25346349","display_name":"Execution model","level":2,"score":0.2678999900817871},{"id":"https://openalex.org/C105446022","wikidata":"https://www.wikidata.org/wiki/Q445962","display_name":"Legacy system","level":3,"score":0.26339998841285706},{"id":"https://openalex.org/C41112130","wikidata":"https://www.wikidata.org/wiki/Q2146175","display_name":"Retiming","level":2,"score":0.26190000772476196},{"id":"https://openalex.org/C101765175","wikidata":"https://www.wikidata.org/wiki/Q577764","display_name":"Communications system","level":2,"score":0.2551000118255615},{"id":"https://openalex.org/C146152329","wikidata":"https://www.wikidata.org/wiki/Q1932543","display_name":"Data migration","level":2,"score":0.2524999976158142}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icpads67057.2025.11323171","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icpads67057.2025.11323171","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 31th International Conference on Parallel and Distributed Systems (ICPADS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W2030553163","https://openalex.org/W2083842231","https://openalex.org/W2166221608","https://openalex.org/W2963786636","https://openalex.org/W2975712713","https://openalex.org/W3081168214","https://openalex.org/W3086105743","https://openalex.org/W3129831491","https://openalex.org/W3189259198","https://openalex.org/W3193250980","https://openalex.org/W3205803342","https://openalex.org/W4386768656"],"related_works":[],"abstract_inverted_index":{"Partially":[0],"sharded":[1],"data":[2,60],"parallel":[3],"(PSDP)":[4],"reduces":[5],"GPU":[6,165],"memory":[7,26],"footprint":[8],"in":[9,93],"data-parallel":[10],"training":[11,158,178],"without":[12],"incurring":[13],"additional":[14],"communication":[15,45,89,112,122,131],"overhead.":[16,124],"Existing":[17],"work":[18],"on":[19,24,163],"PSDP":[20,75,92,192],"has":[21,40],"primarily":[22],"focused":[23],"minimizing":[25],"consumption,":[27],"such":[28],"as":[29],"by":[30,180],"offloading":[31,95],"model":[32,54,109],"states":[33,55],"to":[34,43,63,106,120,151,185],"CPU,":[35],"while":[36,116],"not":[37],"enough":[38],"attention":[39],"been":[41],"paid":[42],"poor":[44,70],"performance.":[46,159],"We":[47,160],"observe":[48],"that":[49,86,134,175],"the":[50,73,88],"imbalanced":[51],"distribution":[52],"of":[53,66,72,91],"across":[56,114],"GPUs":[57],"and":[58,69,96,111],"inter-layer":[59,136],"dependencies":[61],"lead":[62],"severe":[64],"underutilization":[65],"network":[67],"bandwidth":[68,140],"scalability":[71],"current":[74],"system.":[76],"To":[77],"address":[78],"these":[79],"issues,":[80],"we":[81,126,143],"present":[82],"Chimera,":[83],"a":[84,102,128,145],"system":[85],"enhances":[87],"efficiency":[90],"both":[94,108],"non-offloading":[97],"settings.":[98],"First,":[99],"Chimera":[100,162,176],"introduces":[101],"fine-grained":[103,130],"state-sharding":[104],"strategy":[105],"balance":[107],"state":[110],"load":[113],"GPUs,":[115],"leveraging":[117],"tensor":[118,148],"fusion":[119,149,154],"amortize":[121],"launch":[123],"Second,":[125],"design":[127],"multi-stage,":[129],"pipelining":[132],"mechanism":[133],"decouples":[135],"dependencies,":[137],"thereby":[138],"improving":[139],"utilization.":[141],"Finally,":[142],"develop":[144],"practical":[146],"auto-tuning":[147],"algorithm":[150],"adaptively":[152],"optimize":[153],"decisions":[155],"for":[156],"end-to-end":[157],"evaluate":[161],"cloud":[164],"clusters":[166],"using":[167],"representative":[168],"large":[169],"language":[170],"models.":[171],"Experimental":[172],"results":[173],"demonstrate":[174],"improves":[177],"throughput":[179],"<tex":[181,186],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[182,187],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$1.1":[183],"\\times$</tex>":[184,189],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$1.8":[188],"over":[190],"state-of-theart":[191],"systems.":[193]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-01-15T00:00:00"}