{"id":"https://openalex.org/W7161776522","doi":"https://doi.org/10.48550/arxiv.2605.19775","title":"Understanding Inference Scaling for LLMs: Bottlenecks, Trade-offs, and Performance Principles","display_name":"Understanding Inference Scaling for LLMs: Bottlenecks, Trade-offs, and Performance Principles","publication_year":2026,"publication_date":"2026-05-19","ids":{"openalex":"https://openalex.org/W7161776522","doi":"https://doi.org/10.48550/arxiv.2605.19775"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.19775","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.19775","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.19775","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5017707472","display_name":"Moiz Arif","orcid":"https://orcid.org/0000-0002-8823-5562"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Arif, Moiz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136019497","display_name":"Avinash Maurya","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maurya, Avinash","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136519797","display_name":"Sudharshan Vazhkudai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vazhkudai, Sudharshan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136522305","display_name":"Bogdan Nicolae","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nicolae, Bogdan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.32989999651908875,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.32989999651908875,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.2401999980211258,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.0917000025510788,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6654999852180481},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.48750001192092896},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.41019999980926514},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.38589999079704285},{"id":"https://openalex.org/keywords/concurrency","display_name":"Concurrency","score":0.37130001187324524},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.3409999907016754},{"id":"https://openalex.org/keywords/sublinear-function","display_name":"Sublinear function","score":0.33799999952316284},{"id":"https://openalex.org/keywords/programming-paradigm","display_name":"Programming paradigm","score":0.3255000114440918}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7706000208854675},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6654999852180481},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.48750001192092896},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.41019999980926514},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.38589999079704285},{"id":"https://openalex.org/C193702766","wikidata":"https://www.wikidata.org/wiki/Q1414548","display_name":"Concurrency","level":2,"score":0.37130001187324524},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37070000171661377},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3693999946117401},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.3409999907016754},{"id":"https://openalex.org/C117160843","wikidata":"https://www.wikidata.org/wiki/Q338652","display_name":"Sublinear function","level":2,"score":0.33799999952316284},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.3255000114440918},{"id":"https://openalex.org/C2778692605","wikidata":"https://www.wikidata.org/wiki/Q4041866","display_name":"Joins","level":2,"score":0.31779998540878296},{"id":"https://openalex.org/C156884757","wikidata":"https://www.wikidata.org/wiki/Q798554","display_name":"Backtracking","level":2,"score":0.30820000171661377},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.30160000920295715},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.299699991941452},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.2985999882221222},{"id":"https://openalex.org/C43540301","wikidata":"https://www.wikidata.org/wiki/Q689971","display_name":"Paradigm shift","level":2,"score":0.28940001130104065},{"id":"https://openalex.org/C46743427","wikidata":"https://www.wikidata.org/wiki/Q1341685","display_name":"Inference engine","level":3,"score":0.28679999709129333},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.28060001134872437},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.2766000032424927},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.2727999985218048},{"id":"https://openalex.org/C166088908","wikidata":"https://www.wikidata.org/wiki/Q308495","display_name":"Abductive reasoning","level":2,"score":0.2596000134944916},{"id":"https://openalex.org/C201410400","wikidata":"https://www.wikidata.org/wiki/Q1064412","display_name":"Multithreading","level":3,"score":0.25949999690055847}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.19775","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.19775","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.19775","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.19775","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.6436379551887512,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"transition":[1],"from":[2,57,161],"standard":[3,82],"generative":[4],"AI":[5],"to":[6,59],"\\emph{reasoning-centric":[7],"architectures},":[8],"exemplified":[9],"by":[10,29,154],"models":[11,55,96,133,149],"capable":[12],"of":[13,37,184],"extensive":[14],"Chain-of-Thought~(CoT)":[15],"processing,":[16],"marks":[17],"a":[18,44,50,99,167],"fundamental":[19],"paradigm":[20],"shift":[21,41],"in":[22,112],"system":[23,52],"requirements.":[24],"Unlike":[25],"traditional":[26],"workloads":[27,33,104],"dominated":[28],"compute-bound":[30],"prefill,":[31],"reasoning":[32,38,103,174],"generate":[34],"long":[35],"chains":[36],"tokens":[39],"that":[40,80,88],"inference":[42,185],"into":[43],"\\emph{Capacity-Bound":[45],"regime}.":[46],"This":[47],"paper":[48],"presents":[49],"comprehensive":[51],"characterization,":[53],"evaluating":[54],"ranging":[56],"8B":[58],"671B":[60],"parameters":[61],"on":[62,102],"GPUs":[63],"clusters.":[64],"By":[65],"systematically":[66],"exploring":[67],"the":[68,126,173,181],"interplay":[69],"between":[70],"Data,":[71],"Tensor,":[72],"and":[73,121,138,141,156,159],"Pipeline":[74],"parallelism,":[75],"we":[76],"identify":[77],"critical":[78],"bottlenecks":[79],"defy":[81],"scaling":[83],"heuristics.":[84],"Our":[85],"analysis":[86],"reveals":[87],"data":[89],"parallelism":[90,117],"is":[91],"throughput":[92],"efficient":[93],"for":[94,171,180],"small":[95],"but":[97],"hits":[98],"capacity":[100],"trap":[101],"as":[105],"KV-cache":[106],"fragmentation":[107],"forces":[108],"early":[109],"throttling":[110],"resulting":[111],"sub-optimal":[113],"compute":[114],"utilization.":[115],"Tensor":[116],"unlocks":[118],"stranded":[119],"memory":[120],"delivers":[122],"sublinear":[123],"gains":[124],"near":[125],"32B":[127],"crossover.":[128],"At":[129],"frontier":[130],"scale,":[131],"dense":[132],"(e.g.,":[134,150],"Llama-405B)":[135],"are":[136,152],"interconnect":[137],"memory-bandwidth":[139],"bound":[140],"favor":[142],"high-degree":[143],"TP,":[144],"while":[145],"sparse":[146],"Mixture-of-Experts":[147],"(MoE)":[148],"DeepSeek-R1)":[151],"limited":[153],"routing":[155],"synchronization":[157],"latency":[158],"benefit":[160],"hybrid":[162],"strategies.":[163],"These":[164],"insights":[165],"provide":[166],"rigorous":[168],"decision":[169],"framework":[170],"navigating":[172],"cliff,":[175],"establishing":[176],"new":[177],"architectural":[178],"imperatives":[179],"next":[182],"generation":[183],"infrastructure.":[186]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-21T00:00:00"}