{"id":"https://openalex.org/W7207622394","doi":"https://doi.org/10.48550/arxiv.2609.02027","title":"Multi-Turn LLM Conversations under the Least-Recently-Used Policy: Mean-Field Asymptotics and Hit Ratio Approximation","display_name":"Multi-Turn LLM Conversations under the Least-Recently-Used Policy: Mean-Field Asymptotics and Hit Ratio Approximation","publication_year":2026,"publication_date":"2026-09-02","ids":{"openalex":"https://openalex.org/W7207622394","doi":"https://doi.org/10.48550/arxiv.2609.02027"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.02027","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.02027","pdf_url":"https://arxiv.org/pdf/2609.02027","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.02027","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5149921523","display_name":"Heyuan Yao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yao, Heyuan","raw_affiliation_strings":["Northwestern University Evanston , IL 60208"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northwestern University Evanston , IL 60208","institution_ids":["https://openalex.org/I111979921"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052397137","display_name":"Chutong Gao","orcid":"https://orcid.org/0000-0002-3349-353X","observed_orcids":["https://orcid.org/0000-0002-3349-353X"]},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gao, Chutong","raw_affiliation_strings":["Purdue University West Lafayette , IN 47907"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Purdue University West Lafayette , IN 47907","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124449680","display_name":"Yuan Lyu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I4405261251","display_name":"The Hong Kong University of Science and Technology (Guangzhou)","ror":"https://ror.org/050h0vm43","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405261251"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Lyu, Yuan","raw_affiliation_strings":["HKUST Hong Kong SAR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HKUST Hong Kong SAR","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I4405261251"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001892692","display_name":"Isaac Grosof","orcid":"https://orcid.org/0000-0001-6205-8652","observed_orcids":["https://orcid.org/0000-0001-6205-8652"]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]},{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Grosof, Izzy","raw_affiliation_strings":["Northwestern University Evanston , IL 60208","Purdue University West Lafayette , IN 47907"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northwestern University Evanston , IL 60208","institution_ids":["https://openalex.org/I111979921"]},{"raw_affiliation_string":"Purdue University West Lafayette , IN 47907","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112431388","display_name":"David Simchi\u2010Levi","orcid":"https://orcid.org/0000-0002-4650-1519","observed_orcids":["https://orcid.org/0000-0002-4650-1519"]},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Simchi-Levi, David","raw_affiliation_strings":["Purdue University West Lafayette , IN 47907"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Purdue University West Lafayette , IN 47907","institution_ids":["https://openalex.org/I219193219"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.5576000213623047,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.5576000213623047,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.15320000052452087,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.03579999879002571,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multi-turn-dialogue","display_name":"multi-turn dialogue","score":0.7620000243186951},{"id":"https://openalex.org/keywords/large-language-model-serving","display_name":"large language model serving","score":0.9269999861717224},{"id":"https://openalex.org/keywords/kv-cache","display_name":"KV cache","score":0.6549999713897705},{"id":"https://openalex.org/keywords/high-bandwidth-memory","display_name":"high-bandwidth memory","score":0.8579999804496765},{"id":"https://openalex.org/keywords/cache-eviction","display_name":"cache eviction","score":0.6430000066757202}],"concepts":[{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.7053999900817871},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6470000147819519},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.4959999918937683},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.46720001101493835},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.41290000081062317},{"id":"https://openalex.org/C162838799","wikidata":"https://www.wikidata.org/wiki/Q596077","display_name":"Counterexample","level":2,"score":0.3912000060081482},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.37119999527931213},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3711000084877014},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.3492000102996826},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.33719998598098755},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.335999995470047},{"id":"https://openalex.org/C149629883","wikidata":"https://www.wikidata.org/wiki/Q660926","display_name":"Fraction (chemistry)","level":2,"score":0.3285999894142151},{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.32690000534057617},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.31520000100135803},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3001999855041504},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.3000999987125397},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.30000001192092896},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.29030001163482666},{"id":"https://openalex.org/C7366592","wikidata":"https://www.wikidata.org/wiki/Q1255620","display_name":"Dram","level":2,"score":0.2782000005245209},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.2662999927997589},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2524999976158142},{"id":"https://openalex.org/C12186640","wikidata":"https://www.wikidata.org/wiki/Q6815743","display_name":"Memory model","level":3,"score":0.25220000743865967}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.02027","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.02027","pdf_url":"https://arxiv.org/pdf/2609.02027","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.02027","doi":"https://doi.org/10.48550/arxiv.2609.02027","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.02027","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.02027","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.02027","pdf_url":"https://arxiv.org/pdf/2609.02027","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7207622394.pdf","grobid_xml":"https://content.openalex.org/works/W7207622394.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4393853379","https://openalex.org/W4405766252","https://openalex.org/W4415133953","https://openalex.org/W4402155774","https://openalex.org/W7164898985","https://openalex.org/W7150150372","https://openalex.org/W4414243320","https://openalex.org/W7125908883","https://openalex.org/W4408893361","https://openalex.org/W7203662383"],"abstract_inverted_index":{"The":[0,35],"major":[1],"workloads":[2],"in":[3,51],"modern":[4],"large":[5],"language":[6],"model":[7,105,167],"(LLM)":[8],"serving":[9,162,184],"systems":[10,185],"have":[11],"shifted":[12],"from":[13,47],"single-shot":[14],"LLM":[15,161,183],"calls":[16],"to":[17,74,93,130,136],"multi-turn":[18,103,182],"conversations,":[19],"where":[20,79],"new":[21],"responses":[22],"are":[23],"generated":[24],"based":[25],"on":[26,141,164,169],"the":[27,39,65,75,80,99,107,120,125,132,142,145,165,179],"whole":[28],"conversation":[29,104,121],"history":[30],"across":[31],"all":[32],"previous":[33],"turns.":[34],"hit":[36,66,133,152],"ratio,":[37],"i.e.,":[38],"average":[40],"fraction":[41],"of":[42,144,181],"KV":[43,81],"caches":[44,49],"accessed":[45],"directly":[46],"existing":[48],"stored":[50],"high-bandwidth":[52],"memory":[53,95,126,191],"(HBM),":[54],"is":[55,68],"hence":[56],"a":[57,69,102,112,137,150,175,187],"crucial":[58],"metric":[59],"that":[60,118],"governs":[61],"system":[62,77,100],"performance.":[63],"Estimating":[64],"ratio":[67,134,153],"highly":[70],"nontrivial":[71],"task":[72],"due":[73,92],"complex":[76],"dynamics,":[78],"cache":[82],"prefixes":[83],"grow":[84,128],"with":[85],"turns":[86],"and":[87,124,155,186],"some":[88],"must":[89],"be":[90],"evicted":[91],"finite":[94],"capacity.":[96],"We":[97],"formulate":[98],"as":[101,119],"under":[106],"least-recently-used":[108],"(LRU)":[109],"policy.":[110],"Through":[111],"mean-field":[113],"asymptotic":[114],"framework,":[115],"we":[116,147],"prove":[117],"arrival":[122],"rate":[123],"capacity":[127,192],"proportionally":[129],"infinity,":[131],"converges":[135],"closed-form":[138],"limit.":[139],"Based":[140],"characterization":[143],"limit,":[146],"further":[148],"propose":[149],"practical":[151,188],"estimator,":[154],"validate":[156],"its":[157],"accuracy":[158],"by":[159],"real":[160],"experiments":[163],"Qwen3-8B":[166],"implemented":[168],"Ascend":[170],"NPUs.":[171],"Our":[172],"results":[173],"provide":[174],"theoretical":[176],"foundation":[177],"for":[178,190],"analysis":[180],"guideline":[189],"provisioning.":[193]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2026-09-04T00:00:00"}