{"id":"https://openalex.org/W7165656552","doi":"https://doi.org/10.48550/arxiv.2606.22768","title":"Factored Gossip DiLoCo: Reducing Blocking Communication in DiLoCo","display_name":"Factored Gossip DiLoCo: Reducing Blocking Communication in DiLoCo","publication_year":2026,"publication_date":"2026-06-22","ids":{"openalex":"https://openalex.org/W7165656552","doi":"https://doi.org/10.48550/arxiv.2606.22768"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.22768","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.22768","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.22768","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075325374","display_name":"Chamin Hewa Koneputugodage","orcid":"https://orcid.org/0009-0000-1333-5967"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Koneputugodage, Chamin Hewa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039055995","display_name":"Thalaiyasingam Ajanthan","orcid":"https://orcid.org/0000-0002-6431-0775"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ajanthan, Thalaiyasingam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139206494","display_name":"Sameera Ramasinghe","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ramasinghe, Sameera","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024380002","display_name":"Hadi M. Dolatabadi","orcid":"https://orcid.org/0000-0001-9418-1487"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dolatabadi, Hadi Mohaghegh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072026088","display_name":"Shamane Siriwardhana","orcid":"https://orcid.org/0000-0003-0688-7901"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Siriwardhana, Shamane","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136948014","display_name":"Gil Avraham","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Avraham, Gil","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139157544","display_name":"Violetta Shevchenko","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shevchenko, Violetta","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038823661","display_name":"Karol Paj\u0105k","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pajak, Karol","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124904473","display_name":"James Snewin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Snewin, James","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139129268","display_name":"Alexander Long","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Long, Alexander","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.46480000019073486,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.46480000019073486,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10714","display_name":"Software-Defined Networks and 5G","score":0.12200000137090683,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.07840000092983246,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/gossip","display_name":"Gossip","score":0.9057999849319458},{"id":"https://openalex.org/keywords/blocking","display_name":"Blocking (statistics)","score":0.782800018787384},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.7439000010490417},{"id":"https://openalex.org/keywords/mixing","display_name":"Mixing (physics)","score":0.5401999950408936},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5383999943733215},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.5264999866485596},{"id":"https://openalex.org/keywords/call-blocking","display_name":"Call blocking","score":0.43950000405311584},{"id":"https://openalex.org/keywords/ranging","display_name":"Ranging","score":0.3815999925136566}],"concepts":[{"id":"https://openalex.org/C62989814","wikidata":"https://www.wikidata.org/wiki/Q854648","display_name":"Gossip","level":2,"score":0.9057999849319458},{"id":"https://openalex.org/C144745244","wikidata":"https://www.wikidata.org/wiki/Q4927286","display_name":"Blocking (statistics)","level":2,"score":0.782800018787384},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7818999886512756},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.7439000010490417},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5652999877929688},{"id":"https://openalex.org/C138777275","wikidata":"https://www.wikidata.org/wiki/Q6884054","display_name":"Mixing (physics)","level":2,"score":0.5401999950408936},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5383999943733215},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.5264999866485596},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.45239999890327454},{"id":"https://openalex.org/C2775972105","wikidata":"https://www.wikidata.org/wiki/Q5021606","display_name":"Call blocking","level":3,"score":0.43950000405311584},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.3815999925136566},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.37400001287460327},{"id":"https://openalex.org/C2780799671","wikidata":"https://www.wikidata.org/wiki/Q17087362","display_name":"Transient (computer programming)","level":2,"score":0.3718000054359436},{"id":"https://openalex.org/C187834632","wikidata":"https://www.wikidata.org/wiki/Q188804","display_name":"Factorization","level":2,"score":0.35589998960494995},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.32179999351501465},{"id":"https://openalex.org/C3826847","wikidata":"https://www.wikidata.org/wiki/Q188768","display_name":"FLOPS","level":2,"score":0.3077000081539154},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2987000048160553},{"id":"https://openalex.org/C108734733","wikidata":"https://www.wikidata.org/wiki/Q1172333","display_name":"Data synchronization","level":3,"score":0.29649999737739563},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.2784999907016754},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.275299996137619},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2711000144481659},{"id":"https://openalex.org/C48677424","wikidata":"https://www.wikidata.org/wiki/Q6888088","display_name":"Mode (computer interface)","level":2,"score":0.26969999074935913},{"id":"https://openalex.org/C130120984","wikidata":"https://www.wikidata.org/wiki/Q2835898","display_name":"Distributed algorithm","level":2,"score":0.2669999897480011}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.22768","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.22768","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.22768","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.22768","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.4049674868583679}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"To":[0],"make":[1],"large-scale":[2],"distributed":[3],"training":[4,104],"practical":[5],"outside":[6],"high-bandwidth":[7],"datacenters,":[8],"we":[9],"must":[10],"reduce":[11],"blocking,":[12],"high-volume":[13],"synchronization.":[14],"While":[15],"DiLoCo":[16,53],"communicates":[17],"infrequently,":[18],"its":[19],"outer":[20],"synchronization":[21,34,37,54],"remains":[22],"bandwidth-heavy":[23],"and":[24,28,45,66,82,113],"brittle":[25],"to":[26,35,51,87,101,109,117],"stragglers":[27],"transient":[29],"failures.":[30,47,118],"We":[31],"relax":[32],"exact":[33],"approximate":[36],"via":[38],"mixing/gossip,":[39],"which":[40],"degrades":[41],"gracefully":[42],"under":[43],"delays":[44],"communication":[46],"This":[48],"allows":[49],"us":[50],"factorize":[52],"into":[55],"a":[56,67,76],"non-blocking":[57],"mixing":[58,69],"step":[59,70],"that":[60,71],"overlaps":[61],"computation":[62],"with":[63,103],"no":[64],"staleness,":[65],"blocking":[68],"tightens":[72],"worker":[73],"agreement,":[74],"yielding":[75],"tunable":[77],"trade-off":[78],"between":[79],"compute":[80,98],"utilization":[81,99],"optimization":[83],"stability.":[84],"On":[85],"up":[86],"billion-parameter":[88],"language":[89],"models":[90],"in":[91],"low-bandwidth":[92],"settings,":[93],"our":[94],"framework":[95],"substantially":[96],"improves":[97],"compared":[100],"DiLoCo,":[102],"progress":[105],"ranging":[106],"from":[107],"comparable":[108],"closely":[110],"matching":[111],"it,":[112],"is":[114],"more":[115],"robust":[116]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-24T00:00:00"}