{"id":"https://openalex.org/W4406071672","doi":"https://doi.org/10.52202/085713-3773","title":"Ada-KV: Optimizing KV Cache Eviction by Adaptive Budget Allocation for Efficient LLM Inference","display_name":"Ada-KV: Optimizing KV Cache Eviction by Adaptive Budget Allocation for Efficient LLM Inference","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4406071672","doi":"https://doi.org/10.52202/085713-3773"},"language":"en","primary_location":{"id":"doi:10.52202/085713-3773","doi":"https://doi.org/10.52202/085713-3773","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-3773","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.11550","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5062436243","display_name":"Yuan Feng","orcid":"https://orcid.org/0009-0006-6448-9345","observed_orcids":["https://orcid.org/0009-0006-6448-9345","https://orcid.org/0000-0001-9888-4622"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuan Feng","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049757790","display_name":"Junlin Lv","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junlin Lv","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009017779","display_name":"Yukun Cao","orcid":"https://orcid.org/0000-0002-8391-841X","observed_orcids":["https://orcid.org/0000-0002-8391-841X"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yukun Cao","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037366245","display_name":"Xike Xie","orcid":"https://orcid.org/0000-0001-5290-5408","observed_orcids":["https://orcid.org/0000-0001-5290-5408"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xike Xie","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001768407","display_name":"S. Kevin Zhou","orcid":"https://orcid.org/0009-0009-3863-1100","observed_orcids":["https://orcid.org/0009-0009-3863-1100"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"S. Kevin Zhou","raw_affiliation_strings":["CAS"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CAS","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":65.2714,"has_fulltext":true,"cited_by_count":186,"citation_normalized_percentile":{"value":0.99923996,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"125462","last_page":"125498"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5425000190734863,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5425000190734863,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.35109999775886536,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.04450000077486038,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/kv-cache-eviction","display_name":"KV cache eviction","score":0.8149999976158142},{"id":"https://openalex.org/keywords/large-language-model-inference","display_name":"large language model inference","score":0.7580000162124634}],"concepts":[{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.7543883919715881},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6704457998275757},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6576056480407715},{"id":"https://openalex.org/C2778227907","wikidata":"https://www.wikidata.org/wiki/Q1893186","display_name":"Eviction","level":2,"score":0.6165320873260498},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.41882967948913574},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.3455308973789215},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.13005927205085754},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.11844390630722046},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52202/085713-3773","doi":"https://doi.org/10.52202/085713-3773","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-3773","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2407.11550","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.11550","pdf_url":"https://arxiv.org/pdf/2407.11550","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.11550","doi":"https://doi.org/10.48550/arxiv.2407.11550","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.11550","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.11550","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.11550","pdf_url":"https://arxiv.org/pdf/2407.11550","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G7571408944","display_name":null,"funder_award_id":"62271465","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4406071672.pdf","grobid_xml":"https://content.openalex.org/works/W4406071672.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W2948120406","https://openalex.org/W2952862139","https://openalex.org/W2989743967","https://openalex.org/W3015468748","https://openalex.org/W3165895465","https://openalex.org/W3170490008","https://openalex.org/W3171639395","https://openalex.org/W4327810158","https://openalex.org/W4382334332","https://openalex.org/W4384648639","https://openalex.org/W4387294617","https://openalex.org/W4387355950","https://openalex.org/W4387561528","https://openalex.org/W4391766637","https://openalex.org/W4391876619","https://openalex.org/W4392679398","https://openalex.org/W4394778349","https://openalex.org/W4394973219","https://openalex.org/W4399414358","https://openalex.org/W4402671766","https://openalex.org/W4403571350","https://openalex.org/W4403709537","https://openalex.org/W4404340302","https://openalex.org/W4407244915","https://openalex.org/W4414634795","https://openalex.org/W4415689913","https://openalex.org/W4415797413","https://openalex.org/W4416036995","https://openalex.org/W4417136668","https://openalex.org/W4417174708","https://openalex.org/W6910722644","https://openalex.org/W6966825443","https://openalex.org/W7084400772","https://openalex.org/W7131102605"],"related_works":["https://openalex.org/W7172487909","https://openalex.org/W4417410215","https://openalex.org/W7128490381","https://openalex.org/W6891968478","https://openalex.org/W7160820266","https://openalex.org/W4416035932","https://openalex.org/W4407764021","https://openalex.org/W4416035230","https://openalex.org/W4417104132","https://openalex.org/W4416044368"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"have":[3],"excelled":[4],"in":[5],"various":[6],"domains":[7],"but":[8],"face":[9],"efficiency":[10],"challenges":[11],"due":[12],"to":[13,26],"the":[14,56,80,90,103],"growing":[15],"Key-Value":[16],"(KV)":[17],"cache":[18,29,35,85,119],"required":[19],"for":[20],"long-sequence":[21],"inference.":[22],"Recent":[23],"efforts":[24],"aim":[25],"reduce":[27],"KV":[28],"size":[30],"by":[31],"evicting":[32],"vast":[33],"non-critical":[34],"elements":[36],"during":[37],"runtime":[38],"while":[39,88],"preserving":[40],"generation":[41],"quality.":[42],"However,":[43],"these":[44],"methods":[45],"typically":[46],"allocate":[47],"compression":[48],"budgets":[49],"uniformly":[50],"across":[51],"all":[52,134],"attention":[53,58,77],"heads,":[54],"ignoring":[55],"unique":[57],"patterns":[59],"of":[60,83,92],"each":[61],"head.":[62],"In":[63],"this":[64],"paper,":[65],"we":[66,99],"establish":[67],"a":[68],"theoretical":[69],"loss":[70],"upper":[71],"bound":[72],"between":[73],"pre-":[74],"and":[75,129,139],"post-eviction":[76],"output,":[78],"explaining":[79],"optimization":[81,91],"target":[82],"prior":[84,118],"eviction":[86,120],"methods,":[87],"guiding":[89],"adaptive":[93,106],"budget":[94,107],"allocation.":[95],"Base":[96],"on":[97,124],"this,":[98],"propose":[100],"{\\it":[101],"Ada-KV},":[102],"first":[104],"head-wise":[105],"allocation":[108],"strategy.":[109],"It":[110],"offers":[111],"plug-and-play":[112],"benefits,":[113],"enabling":[114],"seamless":[115],"integration":[116],"with":[117],"methods.":[121,148],"Extensive":[122],"evaluations":[123],"13":[125],"datasets":[126,131],"from":[127,132],"Ruler":[128],"16":[130],"LongBench,":[133],"conducted":[135],"under":[136],"both":[137],"question-aware":[138],"question-agnostic":[140],"scenarios,":[141],"demonstrate":[142],"substantial":[143],"quality":[144],"improvements":[145],"over":[146],"existing":[147],"Our":[149],"code":[150],"is":[151],"available":[152],"at":[153],"https://github.com/FFY0/AdaKV.":[154]},"counts_by_year":[{"year":2026,"cited_by_count":124},{"year":2025,"cited_by_count":60},{"year":2024,"cited_by_count":2}],"updated_date":"2026-11-11T12:16:26.678958","created_date":"2025-01-05T00:00:00"}