{"id":"https://openalex.org/W7160930732","doi":"https://doi.org/10.48550/arxiv.2605.09165","title":"Sparse Layers are Critical to Scaling Looped Language Models","display_name":"Sparse Layers are Critical to Scaling Looped Language Models","publication_year":2026,"publication_date":"2026-05-09","ids":{"openalex":"https://openalex.org/W7160930732","doi":"https://doi.org/10.48550/arxiv.2605.09165"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.09165","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09165","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.09165","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135948431","display_name":"Ryan Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Ryan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027237515","display_name":"Jacob Biloki","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Biloki, Jacob","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135993722","display_name":"Edward J. Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Edward J.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135490559","display_name":"Jonathan May","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"May, Jonathan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.46320000290870667,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.46320000290870667,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.07329999655485153,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.05290000140666962,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.6358000040054321},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6176999807357788},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5203999876976013},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4009000062942505},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.3052999973297119},{"id":"https://openalex.org/keywords/trace","display_name":"TRACE (psycholinguistics)","score":0.29660001397132874}],"concepts":[{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6358000040054321},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6176999807357788},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5770000219345093},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5203999876976013},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4009000062942505},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.39250001311302185},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.3052999973297119},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.29660001397132874},{"id":"https://openalex.org/C207390915","wikidata":"https://www.wikidata.org/wiki/Q1230525","display_name":"Divergence (linguistics)","level":2,"score":0.2791999876499176},{"id":"https://openalex.org/C22679943","wikidata":"https://www.wikidata.org/wiki/Q159375","display_name":"Standard deviation","level":2,"score":0.2766000032424927},{"id":"https://openalex.org/C152588345","wikidata":"https://www.wikidata.org/wiki/Q6498959","display_name":"Scale model","level":2,"score":0.2678999900817871},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.2581000030040741},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.25600001215934753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.09165","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09165","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.09165","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09165","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Looped":[0],"language":[1],"models":[2,24,56,66,103],"repeat":[3],"a":[4,146,153],"set":[5],"of":[6],"transformer":[7],"layers":[8,121],"through":[9,87],"depth,":[10],"reducing":[11],"memory":[12,171],"costs":[13],"and":[14,39,44,47,172],"providing":[15],"natural":[16],"early-exit":[17],"points":[18],"at":[19,139,165],"loop":[20,116,127],"boundaries.":[21],"However,":[22],"looped":[23,65,102,151],"do":[25,67],"not":[26,160],"scale":[27,57],"as":[28,30,133],"favorably":[29],"standard":[31,38,61,112,163],"transformers":[32,164],"with":[33,43,108,118,156,175],"unique":[34],"layers.":[35],"We":[36,69],"compare":[37],"Mixture-of-Experts":[40],"(MoE)":[41],"transformers,":[42],"without":[45,94],"looping,":[46],"find":[48,54],"two":[49],"main":[50],"results.":[51],"First,":[52],"we":[53,144],"Looped-MoE":[55,78,154],"better":[58,105],"than":[59,111],"the":[60,88,119,124],"baseline":[62],"while":[63],"dense":[64],"not.":[68],"trace":[70],"this":[71],"to":[72],"routing":[73],"divergence":[74],"between":[75],"loops:":[76],"in":[77,178],"models,":[79],"different":[80],"experts":[81],"are":[82,129],"activated":[83],"on":[84],"each":[85,115],"pass":[86],"same":[89,120],"shared":[90],"layers,":[91],"recovering":[92],"expressivity":[93],"additional":[95],"parameters.":[96],"Our":[97],"second":[98],"finding":[99],"is":[100],"that":[101,122],"have":[104],"compute-quality":[106],"trade-offs":[107],"early":[109,157],"exits":[110,158],"models.":[113],"Because":[114],"ends":[117],"produce":[123],"final":[125],"output,":[126],"boundaries":[128],"superior":[130],"exit":[131],"points,":[132],"confirmed":[134],"by":[135],"earlier":[136],"output":[137],"convergence":[138],"these":[140],"points.":[141],"In":[142],"sum,":[143],"provide":[145],"clear":[147],"direction":[148],"for":[149],"scaling":[150],"models:":[152],"model":[155],"can":[159],"only":[161],"beat":[162],"scale,":[166],"but":[167],"also":[168],"enable":[169],"significant":[170],"inference":[173],"savings":[174],"minimal":[176],"degradation":[177],"quality.":[179]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-13T00:00:00"}