{"id":"https://openalex.org/W7214714124","doi":"https://doi.org/10.48550/arxiv.2609.32663","title":"Distance-KV: Exploiting Relative Distance for Efficient Long-Context Inference","display_name":"Distance-KV: Exploiting Relative Distance for Efficient Long-Context Inference","publication_year":2026,"publication_date":"2026-09-26","ids":{"openalex":"https://openalex.org/W7214714124","doi":"https://doi.org/10.48550/arxiv.2609.32663"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.32663","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.32663","pdf_url":"https://arxiv.org/pdf/2609.32663","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.32663","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136040232","display_name":"Xianpeng Shang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2722730","display_name":"Inner Mongolia University","ror":"https://ror.org/0106qb496","country_code":"CN","type":"education","lineage":["https://openalex.org/I2722730"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shang, Xianpeng","raw_affiliation_strings":["Inner Mongolia University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Inner Mongolia University","institution_ids":["https://openalex.org/I2722730"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113067526","display_name":"Canbin Huang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huang, Canbin","raw_affiliation_strings":["Shenzhen Loop Area Institute 3 Sun Yat-sen"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shenzhen Loop Area Institute 3 Sun Yat-sen","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153492160","display_name":"Jiang Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2722730","display_name":"Inner Mongolia University","ror":"https://ror.org/0106qb496","country_code":"CN","type":"education","lineage":["https://openalex.org/I2722730"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Jiang","raw_affiliation_strings":["Inner Mongolia University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Inner Mongolia University","institution_ids":["https://openalex.org/I2722730"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043801432","display_name":"Tian Lan","orcid":"https://orcid.org/0009-0004-9941-8948","observed_orcids":["https://orcid.org/0009-0004-9941-8948"]},"institutions":[{"id":"https://openalex.org/I22299242","display_name":"Kyoto University","ror":"https://ror.org/02kpeqv85","country_code":"JP","type":"education","lineage":["https://openalex.org/I22299242"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Lan, Tian","raw_affiliation_strings":["Kyoto University","University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kyoto University","institution_ids":["https://openalex.org/I22299242"]},{"raw_affiliation_string":"University","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038549302","display_name":"Qianyi Cai","orcid":"https://orcid.org/0009-0007-0990-5649","observed_orcids":["https://orcid.org/0009-0007-0990-5649"]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I4405261251","display_name":"The Hong Kong University of Science and Technology (Guangzhou)","ror":"https://ror.org/050h0vm43","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405261251"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Cai, Qianyi","raw_affiliation_strings":["The Hong Kong University of Science and Technology (Guangzhou"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology (Guangzhou","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I4405261251"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040062188","display_name":"Xiaojun Quan","orcid":"https://orcid.org/0000-0002-8385-1083","observed_orcids":["https://orcid.org/0000-0002-8385-1083"]},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Quan, Xiaojun","raw_affiliation_strings":["Shenzhen Loop Area Institute 3 Sun Yat-sen"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shenzhen Loop Area Institute 3 Sun Yat-sen","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5153535197","display_name":"Xiangdong Su","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2722730","display_name":"Inner Mongolia University","ror":"https://ror.org/0106qb496","country_code":"CN","type":"education","lineage":["https://openalex.org/I2722730"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Su, Xiangdong","raw_affiliation_strings":["Inner Mongolia University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Inner Mongolia University","institution_ids":["https://openalex.org/I2722730"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4377000033855438,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4377000033855438,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.41040000319480896,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.06610000133514404,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/key-value-cache-compression","display_name":"key-value cache compression","score":0.9350000023841858},{"id":"https://openalex.org/keywords/long-context-inference","display_name":"long-context inference","score":0.8420000076293945},{"id":"https://openalex.org/keywords/relative-distance","display_name":"relative distance","score":0.8629999756813049},{"id":"https://openalex.org/keywords/attention-heads","display_name":"attention heads","score":0.6119999885559082},{"id":"https://openalex.org/keywords/kv-cache-memory-reduction","display_name":"KV cache memory reduction","score":0.699999988079071},{"id":"https://openalex.org/keywords/decoding-delay","display_name":"decoding latency","score":0.8920000195503235}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7979999780654907},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.7516000270843506},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6488000154495239},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.5375000238418579},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5210000276565552},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5019000172615051},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.45969998836517334},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.4530999958515167},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.4426000118255615},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.42100000381469727},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.4180000126361847},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.41589999198913574},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.3409000039100647},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.30570000410079956},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.30070000886917114},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.2913999855518341},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.2906000018119812},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2888000011444092},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.28850001096725464},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.26899999380111694},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.2549000084400177},{"id":"https://openalex.org/C201148951","wikidata":"https://www.wikidata.org/wiki/Q5015976","display_name":"Cache coloring","level":4,"score":0.25440001487731934},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.25380000472068787}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.32663","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.32663","pdf_url":"https://arxiv.org/pdf/2609.32663","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.32663","doi":"https://doi.org/10.48550/arxiv.2609.32663","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.32663","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.32663","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.32663","pdf_url":"https://arxiv.org/pdf/2609.32663","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214714124.pdf","grobid_xml":"https://content.openalex.org/works/W7214714124.grobid-xml"},"referenced_works_count":8,"referenced_works":["https://openalex.org/W2242818861","https://openalex.org/W2964110616","https://openalex.org/W4387321091","https://openalex.org/W4399991982","https://openalex.org/W4401306886","https://openalex.org/W4402671766","https://openalex.org/W4405655184","https://openalex.org/W4406880389"],"related_works":["https://openalex.org/W4405035770","https://openalex.org/W4403928595","https://openalex.org/W7128525388","https://openalex.org/W7167702318","https://openalex.org/W4407806278","https://openalex.org/W7153671136","https://openalex.org/W4414762093","https://openalex.org/W7165737623","https://openalex.org/W7124162274","https://openalex.org/W4414197699"],"abstract_inverted_index":{"The":[0,80],"memory":[1,148],"usage":[2],"and":[3,77,90,96,109,151,180],"decoding":[4,155],"latency":[5],"of":[6,73],"LLM":[7],"inference":[8,185],"grow":[9],"rapidly":[10],"with":[11,46,85],"context":[12],"length.":[13],"To":[14,55],"reduce":[15],"these":[16,161],"costs,":[17],"key-value":[18],"(KV)":[19],"cache":[20,100,123,147],"compression":[21,124,129],"methods":[22],"selectively":[23],"retain":[24],"cached":[25],"states":[26],"based":[27],"on":[28,136],"token":[29],"importance":[30,103],"or":[31],"differences":[32],"in":[33],"attention":[34,53,75],"patterns":[35],"across":[36,92],"heads.":[37],"However,":[38],"we":[39,59],"discover":[40],"that":[41],"retrieval":[42],"capability":[43],"varies":[44],"substantially":[45],"relative":[47,78,157,164],"distance,":[48],"even":[49],"within":[50],"the":[51,70,86,98,116,127],"same":[52],"head.":[54],"exploit":[56],"this":[57],"structure,":[58],"introduce":[60],"Distance-KV,":[61],"which":[62],"learns":[63],"a":[64,153],"static":[65],"KV":[66,99,122,146],"retention":[67],"pattern":[68,81],"over":[69,177],"joint":[71],"space":[72],"layers,":[74],"heads,":[76],"distances.":[79],"is":[82],"learned":[83],"offline":[84],"language":[87],"model":[88],"frozen":[89],"reused":[91],"inputs":[93],"to":[94,133,158],"prune":[95],"compact":[97],"without":[101],"online":[102],"scoring.":[104],"Across":[105],"three":[106],"backbone":[107],"models":[108],"four":[110],"long-context":[111],"benchmarks,":[112],"Distance-KV":[113,144],"consistently":[114],"achieves":[115,152],"best":[117],"overall":[118],"performance":[119],"among":[120],"competing":[121],"methods,":[125],"exceeding":[126],"strongest":[128],"baseline":[130],"by":[131,149],"up":[132],"9.3":[134],"points":[135],"RULER":[137],"at":[138,142],"128K.":[139],"On":[140],"Llama-3.1-8B-Instruct":[141],"128K,":[143],"reduces":[145],"65.4%":[150],"$1.66\\times$":[154],"speedup":[156],"Dense.":[159],"Together,":[160],"results":[162],"identify":[163],"distance":[165],"as":[166],"an":[167],"important":[168],"structural":[169],"dimension":[170],"for":[171,181],"understanding":[172],"how":[173],"LLMs":[174],"retrieve":[175],"information":[176],"long":[178],"contexts":[179],"designing":[182],"more":[183],"efficient":[184],"methods.":[186]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-30T00:00:00"}