{"id":"https://openalex.org/W7171566618","doi":"https://doi.org/10.48550/arxiv.2607.22577","title":"cMoLLM at Scale: Horizontal Scaling Laws for Mixture-of-LLMs","display_name":"cMoLLM at Scale: Horizontal Scaling Laws for Mixture-of-LLMs","publication_year":2026,"publication_date":"2026-06-06","ids":{"openalex":"https://openalex.org/W7171566618","doi":"https://doi.org/10.48550/arxiv.2607.22577"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.22577","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.22577","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.22577","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5143791789","display_name":"Xin Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Xin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5143859290","display_name":"Yemin Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yemin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125027989","display_name":"Mingda Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Mingda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056119409","display_name":"Letian Li","orcid":"https://orcid.org/0000-0003-0539-4072"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Letian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126323476","display_name":"Shuaishuai Cao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Shuaishuai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5143769085","display_name":"Zhengxiao He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Zhengxiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136167187","display_name":"Ryan Dong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Ryan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4404999911785126,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4404999911785126,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.08370000123977661,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.06069999933242798,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.6435999870300293},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.6385999917984009},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5271000266075134},{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.48080000281333923},{"id":"https://openalex.org/keywords/routing","display_name":"Routing (electronic design automation)","score":0.41449999809265137},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.40709999203681946},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.40470001101493835},{"id":"https://openalex.org/keywords/differentiable-function","display_name":"Differentiable function","score":0.39570000767707825},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.36329999566078186}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6603999733924866},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6435999870300293},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6385999917984009},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5271000266075134},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.48080000281333923},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4551999866962433},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.41449999809265137},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.40709999203681946},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.40470001101493835},{"id":"https://openalex.org/C202615002","wikidata":"https://www.wikidata.org/wiki/Q783507","display_name":"Differentiable function","level":2,"score":0.39570000767707825},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3824000060558319},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.36329999566078186},{"id":"https://openalex.org/C205203396","wikidata":"https://www.wikidata.org/wiki/Q612143","display_name":"Bilinear interpolation","level":2,"score":0.3449000120162964},{"id":"https://openalex.org/C2780069185","wikidata":"https://www.wikidata.org/wiki/Q7977945","display_name":"Equivalence (formal languages)","level":2,"score":0.3352000117301941},{"id":"https://openalex.org/C42090638","wikidata":"https://www.wikidata.org/wiki/Q4048907","display_name":"STREAMS","level":2,"score":0.3303000032901764},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.32659998536109924},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3190000057220459},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.3100000023841858},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.2971000075340271},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.29269999265670776},{"id":"https://openalex.org/C2778484313","wikidata":"https://www.wikidata.org/wiki/Q1172540","display_name":"Data stream","level":2,"score":0.2906000018119812},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.28999999165534973},{"id":"https://openalex.org/C89198739","wikidata":"https://www.wikidata.org/wiki/Q3079880","display_name":"Data stream mining","level":2,"score":0.2896000146865845},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.2849999964237213},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2687000036239624},{"id":"https://openalex.org/C73000952","wikidata":"https://www.wikidata.org/wiki/Q17007827","display_name":"Discretization","level":2,"score":0.26660001277923584},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.260699987411499}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.22577","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.22577","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.22577","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.22577","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Scaling":[0],"large":[1],"language":[2,153],"models":[3,36,147],"(LLMs)":[4],"has":[5],"driven":[6],"their":[7],"success,":[8],"yet":[9],"dense":[10],"Transformers":[11],"couple":[12],"capacity":[13,44],"and":[14,25,66,73,78,81,93,118,156,159,172,178],"computation:":[15],"every":[16,21],"parameter":[17],"is":[18],"activated":[19],"for":[20],"token,":[22],"making":[23],"training":[24],"inference":[26],"costs":[27],"grow":[28],"linearly":[29],"with":[30,165],"model":[31],"size-a":[32],"critical":[33],"bottleneck":[34],"as":[35,105],"approach":[37],"trillion-parameter":[38],"regimes.":[39],"We":[40,96],"aim":[41],"to":[42,113,176],"scale":[43],"through":[45,140],"MoE-style":[46,99],"mixture":[47,100],"throughout":[48],"the":[49,55],"LLM":[50],"pipeline":[51],"rather":[52],"than":[53],"only":[54],"FFN.":[56],"Prior":[57],"pipeline-level":[58],"approaches":[59],"include":[60],"ParaScale,":[61],"which":[62,83],"introduces":[63],"virtual":[64],"tokens":[65],"parallel":[67],"streams":[68,139],"but":[69,89],"incurs":[70],"substantial":[71],"overhead":[72],"suffers":[74],"from":[75],"homogenized":[76],"routing":[77,119],"gradient":[79],"collapse,":[80],"AltUp,":[82],"uses":[84],"an":[85],"auxiliary":[86],"prediction":[87],"branch":[88],"offers":[90],"limited":[91],"adaptivity":[92],"slow":[94],"convergence.":[95],"establish":[97],"that":[98,135],"layers":[101],"can":[102],"be":[103],"reformulated":[104],"variable-kernel":[106],"dynamic":[107,143],"convolutions,":[108],"where":[109],"each":[110],"expert":[111],"corresponds":[112],"a":[114,131],"$1{\\times}1$":[115],"convolutional":[116],"kernel":[117,122],"implements":[120],"input-conditioned":[121],"aggregation.":[123],"Building":[124],"on":[125,149],"this":[126],"equivalence,":[127],"we":[128],"introduce":[129],"cMoLLM:":[130],"convolutionally":[132],"gated":[133],"mixture-of-LLMs":[134],"routes":[136],"over":[137],"end-to-end":[138],"fully":[141],"differentiable":[142],"convolution.":[144],"In":[145],"GPT-2-style":[146],"trained":[148],"FineWeb,":[150],"cMoLLM":[151],"improves":[152],"modeling":[154],"perplexity":[155],"downstream":[157],"GLUE":[158],"SQuAD":[160],"accuracy":[161],"under":[162],"matched":[163],"compute,":[164],"better":[166],"stream":[167],"utilization,":[168],"more":[169],"stable":[170],"optimization,":[171],"favorable":[173],"scaling":[174],"compared":[175],"ParaScale-":[177],"AltUp-style":[179],"baselines.":[180]},"counts_by_year":[],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2026-07-29T00:00:00"}