{"id":"https://openalex.org/W7162677396","doi":"https://doi.org/10.48550/arxiv.2605.27678","title":"Heterogeneous Parallelism for Multimodal Large Language Model Training","display_name":"Heterogeneous Parallelism for Multimodal Large Language Model Training","publication_year":2026,"publication_date":"2026-05-26","ids":{"openalex":"https://openalex.org/W7162677396","doi":"https://doi.org/10.48550/arxiv.2605.27678"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.27678","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27678","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.27678","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037977602","display_name":"Yashaswi Karnati","orcid":"https://orcid.org/0000-0002-2512-1250"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karnati, Yashaswi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137281913","display_name":"Kamran Jafari","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jafari, Kamran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089523268","display_name":"Akash Mehra","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mehra, Akash","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137221995","display_name":"Li Ding","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121328175","display_name":"Pranav Prashant Thombre","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thombre, Pranav Prashant","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137203091","display_name":"Ali Roshan Ghias","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ghias, Ali Roshan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137303099","display_name":"Shifang Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Shifang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120304508","display_name":"Parth Mannan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mannan, Parth","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137268846","display_name":"Yu Yao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137290789","display_name":"Hao Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121303471","display_name":"Eric Harper","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Harper, Eric","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137214040","display_name":"Ashwath Aithal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aithal, Ashwath","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5011019569","display_name":"Nima Tajbakhsh","orcid":"https://orcid.org/0000-0001-8614-4811"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tajbakhsh, Nima","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.34850001335144043,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.34850001335144043,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.17829999327659607,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.1143999993801117,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5667999982833862},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.4747999906539917},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.4731000065803528},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.46950000524520874},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.3801000118255615},{"id":"https://openalex.org/keywords/abstraction","display_name":"Abstraction","score":0.3718999922275543},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.36880001425743103},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.35929998755455017},{"id":"https://openalex.org/keywords/reachability","display_name":"Reachability","score":0.32409998774528503}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8618000149726868},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5667999982833862},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.4747999906539917},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4731000065803528},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.46950000524520874},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4090999960899353},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.3801000118255615},{"id":"https://openalex.org/C124304363","wikidata":"https://www.wikidata.org/wiki/Q673661","display_name":"Abstraction","level":2,"score":0.3718999922275543},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.36880001425743103},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.36309999227523804},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.35929998755455017},{"id":"https://openalex.org/C136643341","wikidata":"https://www.wikidata.org/wiki/Q1361526","display_name":"Reachability","level":2,"score":0.32409998774528503},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.31450000405311584},{"id":"https://openalex.org/C45340560","wikidata":"https://www.wikidata.org/wiki/Q215382","display_name":"Disjoint sets","level":2,"score":0.30790001153945923},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.3066999912261963},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3001999855041504},{"id":"https://openalex.org/C44210515","wikidata":"https://www.wikidata.org/wiki/Q16968978","display_name":"Bespoke","level":2,"score":0.29789999127388},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2921999990940094},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.28690001368522644},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.2806999981403351},{"id":"https://openalex.org/C160403385","wikidata":"https://www.wikidata.org/wiki/Q220543","display_name":"Queue","level":2,"score":0.28040000796318054},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2786000072956085},{"id":"https://openalex.org/C53833338","wikidata":"https://www.wikidata.org/wiki/Q1061424","display_name":"Context switch","level":2,"score":0.2766000032424927},{"id":"https://openalex.org/C2776834041","wikidata":"https://www.wikidata.org/wiki/Q25346349","display_name":"Execution model","level":2,"score":0.272599995136261},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.27160000801086426},{"id":"https://openalex.org/C205606062","wikidata":"https://www.wikidata.org/wiki/Q5249645","display_name":"Decoupling (probability)","level":2,"score":0.2687000036239624},{"id":"https://openalex.org/C171018156","wikidata":"https://www.wikidata.org/wiki/Q7370306","display_name":"Rotation formalisms in three dimensions","level":2,"score":0.26109999418258667},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.257099986076355},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.25699999928474426},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.25119999051094055},{"id":"https://openalex.org/C28034677","wikidata":"https://www.wikidata.org/wiki/Q17092530","display_name":"Interleaving","level":2,"score":0.25}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.27678","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27678","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.27678","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27678","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.5466451644897461,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Foundation":[0],"model":[1,87],"training":[2],"is":[3,57,67,120],"becoming":[4],"multimodal,":[5],"from":[6],"post-training":[7],"pipelines":[8],"to":[9,36,144,184,206,217,223],"large-scale":[10],"pretraining.":[11],"As":[12],"modality":[13],"coverage":[14],"broadens,":[15],"context":[16,65],"windows":[17],"grow,":[18],"and":[19,40,101,110,157,174,181,189,219,233],"encoder":[20,48,75],"LLM":[21,53,64],"scales":[22,183],"diverge,":[23],"a":[24,193],"single":[25],"LLM-centric":[26],"TP/CP/PP/DP/EP":[27],"layout":[28,159,188],"increasingly":[29],"limits":[30],"throughput.":[31],"This":[32],"coupling":[33],"forces":[34],"encoders":[35],"inherit":[37],"LLM-driven":[38],"sharding":[39],"placement":[41,166,190],"choices":[42],"that":[43,91,154],"can":[44],"add":[45],"communication,":[46],"limit":[47],"parallelism,":[49],"or":[50],"constrain":[51],"the":[52,55,70,134,145,235],"schedule;":[54],"mismatch":[56],"most":[58],"pronounced":[59],"at":[60],"long":[61],"contexts,":[62],"where":[63],"parallelism":[66,82],"needed":[68],"for":[69,83,133,164],"fused":[71],"multimodal":[72,84,179],"sequence":[73],"but":[74],"inputs":[76],"remain":[77],"bounded.":[78],"We":[79,148,168,225],"present":[80],"heterogeneous":[81,176],"large":[85],"language":[86],"training,":[88],"an":[89,238],"abstraction":[90],"lets":[92],"modules":[93],"in":[94],"one":[95],"end-to-end":[96],"graph":[97],"use":[98],"independent":[99,126],"layouts":[100],"rank":[102,115],"placements,":[103],"supporting":[104],"colocated":[105,172,200],"execution":[106,112],"on":[107,113],"shared":[108],"GPUs":[109],"non-colocated":[111,175,209],"disjoint":[114],"sets.":[116],"The":[117],"key":[118],"challenge":[119],"preserving":[121],"boundary":[122,152],"tensor":[123],"semantics":[124],"across":[125,178],"layouts:":[127],"forward":[128,156],"activations":[129],"must":[130,140],"be":[131,141],"materialized":[132],"destination":[135],"layout,":[136],"while":[137,208],"backward":[138,158],"gradients":[139],"routed":[142],"back":[143],"source":[146],"layout.":[147],"address":[149],"this":[150,198],"with":[151],"communicators":[153],"implement":[155],"transforms,":[160],"plus":[161],"scheduling":[162],"extensions":[163],"both":[165],"modes.":[167],"evaluate":[169],"optimized":[170],"homogeneous,":[171],"heterogeneous,":[173],"configurations":[177],"workloads":[180],"GPU":[182],"characterize":[185],"when":[186],"added":[187],"freedom":[191],"exposes":[192],"better":[194],"operating":[195],"point.":[196],"Across":[197],"sweep,":[199],"heterogeneity":[201,210],"improves":[202,211],"TFLOPS/GPU":[203,220],"by":[204,215,221],"up":[205,216,222],"49.3%,":[207],"aggregate":[212],"token":[213],"throughput":[214],"13.0%":[218],"9.6%.":[224],"validate":[226],"loss":[227],"convergence":[228],"parity":[229],"against":[230],"homogeneous":[231],"baselines":[232],"release":[234],"system":[236],"as":[237],"open-source":[239],"Megatron-LM":[240],"extension.":[241]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-29T00:00:00"}