{"id":"https://openalex.org/W7225268175","doi":"https://doi.org/10.48550/arxiv.2611.05150","title":"What Sets the Serving Speed of Looped Language Models? Measurements and Optimizations Graded by Exactness","display_name":"What Sets the Serving Speed of Looped Language Models? Measurements and Optimizations Graded by Exactness","publication_year":2026,"publication_date":"2026-11-03","ids":{"openalex":"https://openalex.org/W7225268175","doi":"https://doi.org/10.48550/arxiv.2611.05150"},"language":"en","primary_location":{"id":"doi:10.48550/arxiv.2611.05150","doi":"https://doi.org/10.48550/arxiv.2611.05150","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.05150","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2611.05150","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5159205575","display_name":"Dezhen Lu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Dezhen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066706242","display_name":"Runguo Li","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Runguo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123363351","display_name":"Zixiang Ni","orcid":"https://orcid.org/0009-0009-6008-6735","observed_orcids":["https://orcid.org/0009-0009-6008-6735"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ni, Zixiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160653551","display_name":"Yuchen Tang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Yuchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001887923","display_name":"\u5b54\u884c\u6743","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kong, Xingquan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101605462","display_name":"Jiaxing Yang","orcid":"https://orcid.org/0009-0005-8439-1428","observed_orcids":["https://orcid.org/0009-0005-8439-1428"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Jiaxing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160604802","display_name":"Jiarui Guo","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Jiarui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5160333580","display_name":"Lingxiang Gao","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Lingxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.6664999723434448,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.6664999723434448,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1266999989748001,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.05820000171661377,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/throughput-optimization","display_name":"throughput maximization","score":0.6510000228881836},{"id":"https://openalex.org/keywords/kernel-fusion","display_name":"kernel fusion","score":0.5180000066757202},{"id":"https://openalex.org/keywords/gsm8k","display_name":"GSM8K","score":0.6019999980926514}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6782000064849854},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.6456999778747559},{"id":"https://openalex.org/C2776175482","wikidata":"https://www.wikidata.org/wiki/Q1195816","display_name":"Transfer (computing)","level":2,"score":0.5027999877929688},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.4893999993801117},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.48410001397132874},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.47099998593330383},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.46389999985694885},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.46309998631477356},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.45680001378059387},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.44130000472068787},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.4244000017642975},{"id":"https://openalex.org/C153180980","wikidata":"https://www.wikidata.org/wiki/Q19776675","display_name":"Commit","level":2,"score":0.40939998626708984},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.4043999910354614},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.35249999165534973},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.33090001344680786},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.323199987411499},{"id":"https://openalex.org/C136625980","wikidata":"https://www.wikidata.org/wiki/Q663208","display_name":"Rounding","level":2,"score":0.2953999936580658},{"id":"https://openalex.org/C41431624","wikidata":"https://www.wikidata.org/wiki/Q1053357","display_name":"Block size","level":3,"score":0.28679999709129333},{"id":"https://openalex.org/C84211073","wikidata":"https://www.wikidata.org/wiki/Q117879","display_name":"Floating point","level":2,"score":0.27889999747276306},{"id":"https://openalex.org/C135135165","wikidata":"https://www.wikidata.org/wiki/Q386456","display_name":"Token bucket","level":3,"score":0.2711000144481659},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.26899999380111694}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2611.05150","doi":"https://doi.org/10.48550/arxiv.2611.05150","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.05150","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2611.05150","doi":"https://doi.org/10.48550/arxiv.2611.05150","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.05150","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7220926987","https://openalex.org/W7219155330","https://openalex.org/W7220986529","https://openalex.org/W6903721849","https://openalex.org/W4413059409","https://openalex.org/W7224437624","https://openalex.org/W7167483051","https://openalex.org/W7161247150","https://openalex.org/W4402698921","https://openalex.org/W7221941876"],"abstract_inverted_index":{"Looped":[0],"language":[1],"models":[2],"apply":[3],"one":[4,145,232,243],"block":[5],"of":[6,181,220],"layers":[7],"many":[8],"times":[9],"per":[10,21,65],"token,":[11],"so":[12,106],"a":[13,93,194],"serving":[14,25],"engine":[15,34],"pays":[16],"every":[17],"per-layer":[18],"overhead":[19],"once":[20],"loop;":[22],"we":[23,107],"measure":[24],"for":[26],"Ouro":[27,57,130],"and":[28,35,71,91,101,122,133,140,193,216],"Huginn-0125":[29],"in":[30,218,235],"an":[31,46],"open-source":[32],"looped-model":[33],"grade":[36],"our":[37],"optimizations":[38],"from":[39,241],"bitwise-identical":[40],"changes":[41,142],"to":[42,206,230,249],"those":[43],"that":[44,163,259],"require":[45],"accuracy":[47],"check.":[48],"At":[49],"batch":[50],"1,":[51],"fusing":[52,72],"all":[53,148],"loop":[54,188],"submissions":[55],"leaves":[56,179],"throughput":[58,78,210],"at":[59],"0.99x;":[60],"the":[61,68,127,159,208,238,246,256,260],"5689":[62],"small":[63],"kernels":[64],"token":[66],"are":[67],"real":[69],"cost,":[70],"them":[73],"behind":[74],"rounding":[75],"fences":[76],"raises":[77],"1.11-1.59x":[79],"while":[80,137,187],"1014/1014":[81],"requests":[82],"remain":[83],"identical.":[84],"An":[85],"approximate":[86],"fusion":[87,189],"is":[88],"no":[89],"faster":[90,125],"misses":[92],"one-point":[94],"GSM8K":[95],"rule":[96,164],"on":[97,147,158,166],"87":[98],"questions":[99],"(61":[100],"62":[102],"correct":[103],"versus":[104],"64),":[105],"do":[108],"not":[109],"ship":[110],"it.":[111],"Huginn's":[112,138,153],"released":[113],"defaults":[114],"avoid":[115],"dead":[116],"KV":[117,234],"slots,":[118],"reuse":[119,175],"prefix-cache":[120],"pages,":[121],"serve":[123],"1.77-3.16x":[124],"than":[126],"base":[128,209],"engine;":[129],"reaches":[131,204],"1.65-2.15x":[132],"agent":[134],"sessions":[135],"1.91-3.21x,":[136],"noise":[139,171],"prefix-reuse":[141],"stay":[143],"within":[144],"point":[146,168],"1319":[149],"Huginn":[150],"test":[151],"questions;":[152],"packed":[154],"prefill,":[155],"-0.61":[156],"points":[157],"same":[160],"questions,":[161],"meets":[162],"only":[165],"its":[167],"estimate.":[169],"Position-keyed":[170],"makes":[172],"CPU":[173],"cache":[174,196],"exact":[176],"but":[177],"still":[178],"20":[180],"128":[182],"GPU":[183],"full":[184],"hits":[185],"different,":[186],"delays":[190],"prefill":[191],"admission":[192],"64-graph":[195],"prevents":[197],"eager":[198],"fallbacks.":[199],"Under":[200],"prefill-decode":[201],"disaggregation":[202],"ExactLoop":[203],"up":[205],"3.3x":[207],"with":[211,223],"peer":[212],"error":[213],"handling":[214],"disabled,":[215],"3.1x":[217],"two":[219],"three":[221],"runs":[222],"it":[224],"enabled.":[225],"Its":[226],"transfer":[227],"budget,":[228],"sized":[229],"hold":[231],"request's":[233],"flight,":[236],"cuts":[237],"commit":[239],"tail":[240],"about":[242],"second":[244],"under":[245],"earlier":[247],"budget":[248],"16":[250],"ms":[251],"(base:":[252],"2.8":[253],"s),":[254],"removing":[255],"polling":[257],"rounds":[258],"engine's":[261],"interval":[262],"would":[263],"otherwise":[264],"impose.":[265]},"counts_by_year":[],"updated_date":"2026-11-06T12:05:46.655746","created_date":"2026-11-05T00:00:00"}