{"id":"https://openalex.org/W7131678320","doi":"https://doi.org/10.48550/arxiv.2602.21788","title":"Efficient Scaling of LLM Training with Flexible Context Parallelism","display_name":"Efficient Scaling of LLM Training with Flexible Context Parallelism","publication_year":2026,"publication_date":"2026-02-25","ids":{"openalex":"https://openalex.org/W7131678320","doi":"https://doi.org/10.48550/arxiv.2602.21788"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.21788","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5126878784","display_name":"Yifan Niu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Niu, Yifan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126896864","display_name":"Han Xiao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Han","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126870421","display_name":"Dongyi Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Dongyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126948996","display_name":"Wei Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5126880567","display_name":"Jia Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Jia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.2273358,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.1761000007390976,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.1761000007390976,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.15719999372959137,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.08730000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.8105999827384949},{"id":"https://openalex.org/keywords/data-parallelism","display_name":"Data parallelism","score":0.7436000108718872},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.6531000137329102},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.5806000232696533},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.5612999796867371},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.5458999872207642}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8169000148773193},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.8105999827384949},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.7436000108718872},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6777999997138977},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.6531000137329102},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.5806000232696533},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.5612999796867371},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.5458999872207642},{"id":"https://openalex.org/C140763907","wikidata":"https://www.wikidata.org/wiki/Q2714055","display_name":"Instruction-level parallelism","level":3,"score":0.5450000166893005},{"id":"https://openalex.org/C42992933","wikidata":"https://www.wikidata.org/wiki/Q691169","display_name":"Task parallelism","level":3,"score":0.48069998621940613},{"id":"https://openalex.org/C106515295","wikidata":"https://www.wikidata.org/wiki/Q26806595","display_name":"Parallel processing","level":2,"score":0.4244999885559082},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.3822000026702881},{"id":"https://openalex.org/C201410400","wikidata":"https://www.wikidata.org/wiki/Q1064412","display_name":"Multithreading","level":3,"score":0.33719998598098755},{"id":"https://openalex.org/C2778915421","wikidata":"https://www.wikidata.org/wiki/Q3643177","display_name":"Performance improvement","level":2,"score":0.2574000060558319},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.25450000166893005}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.21788","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.21788","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.21788","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.21788","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Scaling":[0],"long-context":[1],"capabilities":[2],"is":[3,97],"crucial":[4],"for":[5,25],"Large":[6],"Language":[7],"Models":[8],"(LLMs).":[9],"However,":[10],"real-world":[11],"data":[12,45,107],"contain":[13],"a":[14,81],"large":[15],"number":[16],"of":[17],"sequences":[18],"with":[19,89],"heterogeneous":[20],"lengths.":[21],"Existing":[22],"training":[23,94],"libraries":[24],"LLMs":[26],"rely":[27],"on":[28],"static":[29],"parallelism":[30,58,67,77,87],"strategies,":[31],"which":[32],"suffer":[33],"from":[34],"severe":[35],"load":[36],"imbalance,":[37],"redundant":[38],"communication,":[39],"and":[40,65,79,117,122],"suboptimal":[41],"hardware":[42,102],"utilization":[43],"under":[44,105],"heterogeneity.":[46,108],"In":[47],"this":[48],"work,":[49],"we":[50],"propose":[51],"Flexible":[52],"Context":[53],"Parallelism":[54],"(FCP),":[55],"an":[56],"efficient":[57],"strategy":[59],"that":[60,112],"adaptively":[61],"reconfigures":[62],"communication":[63],"groups":[64],"context":[66],"degrees":[68,78],"during":[69],"LLM":[70,121],"training.":[71],"We":[72],"generalize":[73],"more":[74],"flexible":[75],"non-power-of-two":[76],"develop":[80],"polynomial-time":[82],"algorithm":[83],"to":[84,99,127],"generate":[85],"near-optimal":[86],"strategies":[88],"only":[90],"millisecond-level":[91],"overhead":[92],"per":[93],"batch.":[95],"FCP":[96,113,145],"able":[98],"maintain":[100],"high":[101],"efficiency":[103,137],"even":[104,146],"extreme":[106],"Experimental":[109],"results":[110],"demonstrate":[111],"significantly":[114],"outperforms":[115],"Megatron-LM":[116],"DeepSpeed":[118],"in":[119,130],"both":[120],"MLLM":[123],"training,":[124],"achieving":[125],"up":[126],"1.46x":[128],"speedup":[129],"average":[131],"throughput":[132],"while":[133],"maintaining":[134],"near-linear":[135],"scaling":[136],"across":[138],"large-scale":[139],"clusters.":[140],"For":[141],"extremely":[142],"unbalanced":[143],"batches,":[144],"achieves":[147],"2.24x":[148],"speedup.":[149]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-02-27T00:00:00"}