{"id":"https://openalex.org/W4407764366","doi":"https://doi.org/10.48550/arxiv.2502.13542","title":"Activation-aware Probe-Query: Effective Key-Value Retrieval for Long-Context LLMs Inference","display_name":"Activation-aware Probe-Query: Effective Key-Value Retrieval for Long-Context LLMs Inference","publication_year":2025,"publication_date":"2025-02-19","ids":{"openalex":"https://openalex.org/W4407764366","doi":"https://doi.org/10.48550/arxiv.2502.13542"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.13542","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.13542","pdf_url":"https://arxiv.org/pdf/2502.13542","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.13542","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5112964246","display_name":"Qingfa Xiao","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Qingfa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013205685","display_name":"Jiachuan Wang","orcid":"https://orcid.org/0000-0001-6473-8221","observed_orcids":["https://orcid.org/0000-0001-6473-8221"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jiachuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100369546","display_name":"Haoyang Li","orcid":"https://orcid.org/0000-0003-4231-684X","observed_orcids":["https://orcid.org/0000-0003-4231-684X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Haoyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015874725","display_name":"Cheng Dan Deng","orcid":"https://orcid.org/0000-0003-2620-3247","observed_orcids":["https://orcid.org/0000-0003-2620-3247"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deng, Cheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101643367","display_name":"Jiaqi Tang","orcid":"https://orcid.org/0000-0003-4089-0039","observed_orcids":["https://orcid.org/0000-0003-4089-0039"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Jiaqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079920323","display_name":"Shuangyin Li","orcid":"https://orcid.org/0000-0001-6404-3438","observed_orcids":["https://orcid.org/0000-0001-6404-3438"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Shuangyin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102989996","display_name":"Yongqi Zhang","orcid":"https://orcid.org/0000-0003-2085-7418","observed_orcids":["https://orcid.org/0000-0003-2085-7418"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yongqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112977076","display_name":"Jun Wang","orcid":"https://orcid.org/0009-0002-7805-8491","observed_orcids":["https://orcid.org/0009-0002-7805-8491"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100333516","display_name":"Lei Chen","orcid":"https://orcid.org/0000-0002-8257-5806","observed_orcids":["https://orcid.org/0000-0002-8257-5806","https://orcid.org/0000-0002-1054-5501","https://orcid.org/0000-0002-4912-3293","https://orcid.org/0000-0002-7918-8936","https://orcid.org/0000-0003-4664-0700"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Lei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6401000022888184,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6401000022888184,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2549000084400177,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.025299999862909317,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/long-context-llm-inference","display_name":"long-context LLM inference","score":0.7850000262260437},{"id":"https://openalex.org/keywords/sliding-window","display_name":"sliding window","score":0.4749999940395355},{"id":"https://openalex.org/keywords/gpu-memory-efficiency","display_name":"GPU memory efficiency","score":0.6539999842643738},{"id":"https://openalex.org/keywords/training-free-inference","display_name":"training-free inference","score":0.6039999723434448},{"id":"https://openalex.org/keywords/attention-allocation","display_name":"attention allocation","score":0.3709999918937683}],"concepts":[{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7273237705230713},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.7046803832054138},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.662351667881012},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5452440977096558},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5158602595329285},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.513032078742981},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3546519875526428},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.19974055886268616},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.1962880790233612},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.168247252702713},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.09598341584205627},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.13542","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.13542","pdf_url":"https://arxiv.org/pdf/2502.13542","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.13542","doi":"https://doi.org/10.48550/arxiv.2502.13542","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.13542","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.13542","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.13542","pdf_url":"https://arxiv.org/pdf/2502.13542","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4399794499","https://openalex.org/W4415101136","https://openalex.org/W7162449816","https://openalex.org/W7116895983","https://openalex.org/W4405562071","https://openalex.org/W7165737623","https://openalex.org/W4417174708","https://openalex.org/W4416034604","https://openalex.org/W4411403450","https://openalex.org/W4412945657"],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2,11,93],"large":[3],"language":[4],"models":[5],"(LLMs)":[6],"have":[7],"showcased":[8],"exceptional":[9],"performance":[10,196],"long-context":[12],"tasks,":[13],"while":[14],"facing":[15],"significant":[16],"inference":[17,199],"efficiency":[18],"challenges":[19],"with":[20,197],"limited":[21],"GPU":[22],"memory.":[23],"Existing":[24],"solutions":[25],"first":[26],"proposed":[27],"the":[28,55,74,100,128,147,160,166,183,188],"sliding-window":[29],"approach":[30,106,118],"to":[31,54,66,87,97,126],"accumulate":[32],"a":[33,60,115,137,171],"set":[34],"of":[35,150],"historical":[36],"\\textbf{key-value}":[37],"(KV)":[38],"pairs":[39,131,163],"for":[40,132,152],"reuse,":[41],"then":[42],"further":[43],"improvements":[44],"selectively":[45],"retain":[46],"its":[47,194],"subsets":[48],"at":[49,154,182],"each":[50,94,143],"step.":[51],"However,":[52],"due":[53],"sparse":[56],"attention":[57,75],"distribution":[58],"across":[59,180],"long":[61],"context,":[62,102],"it":[63,85,125],"is":[64,76,104],"hard":[65],"identify":[67],"and":[68,123,164,190,201],"recall":[69,159],"relevant":[70,129,161],"KV":[71,162,173],"pairs,":[72],"as":[73,91],"distracted":[77],"by":[78,108,177],"massive":[79],"candidate":[80],"pairs.":[81],"Additionally,":[82],"we":[83,112,169],"found":[84],"promising":[86],"select":[88],"representative":[89],"tokens":[90],"probe-Query":[92,151],"sliding":[95],"window":[96],"effectively":[98],"represent":[99],"entire":[101],"which":[103],"an":[105],"overlooked":[107],"existing":[109],"methods.":[110],"Thus,":[111],"propose":[113],"\\textbf{ActQKV},":[114],"training-free,":[116],"\\textbf{Act}ivation-aware":[117],"that":[119],"dynamically":[120],"determines":[121],"probe-\\textbf{Q}uery":[122],"leverages":[124],"retrieve":[127],"\\textbf{KV}":[130],"inference.":[133],"Specifically,":[134],"ActQKV":[135],"monitors":[136],"token-level":[138],"indicator,":[139],"Activation":[140],"Bias,":[141],"within":[142],"context":[144],"window,":[145],"enabling":[146],"proper":[148],"construction":[149],"retrieval":[153],"pre-filling":[155],"stage.":[156,185],"To":[157],"accurately":[158],"minimize":[165],"irrelevant":[167],"ones,":[168],"design":[170],"dynamic":[172],"cut-off":[174],"mechanism":[175],"guided":[176],"information":[178],"density":[179],"layers":[181],"decoding":[184],"Experiments":[186],"on":[187],"Long-Bench":[189],"$\\infty$":[191],"Benchmarks":[192],"demonstrate":[193],"state-of-the-art":[195],"competitive":[198],"quality":[200],"resource":[202],"efficiency.":[203]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2025-02-21T00:00:00"}