{"id":"https://openalex.org/W7161300701","doi":"https://doi.org/10.48550/arxiv.2605.14037","title":"Self-Pruned Key-Value Attention: Learning When to Write by Predicting Future Utility","display_name":"Self-Pruned Key-Value Attention: Learning When to Write by Predicting Future Utility","publication_year":2026,"publication_date":"2026-05-13","ids":{"openalex":"https://openalex.org/W7161300701","doi":"https://doi.org/10.48550/arxiv.2605.14037"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.14037","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14037","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.14037","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092998290","display_name":"Gergely Szilvasy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Szilvasy, Gergely","raw_affiliation_strings":["Meta FAIR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta FAIR","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093123135","display_name":"Manuel Faysse","orcid":null},"institutions":[{"id":"https://openalex.org/I4210107720","display_name":"CentraleSup\u00e9lec","ror":"https://ror.org/019tcpt25","country_code":"FR","type":"facility","lineage":["https://openalex.org/I277688954","https://openalex.org/I4210107720"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Faysse, Manuel","raw_affiliation_strings":["Meta FAIR","MICS, CentraleSup\u00e9lec"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta FAIR","institution_ids":[]},{"raw_affiliation_string":"MICS, CentraleSup\u00e9lec","institution_ids":["https://openalex.org/I4210107720"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002000099","display_name":"Mar\u00eda Lomel\u00ed","orcid":"https://orcid.org/0000-0002-6524-3615"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lomeli, Maria","raw_affiliation_strings":["Meta FAIR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta FAIR","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136257586","display_name":"Matthijs Douze","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Douze, Matthijs","raw_affiliation_strings":["Meta FAIR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta FAIR","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040376974","display_name":"Pierre-Emmanuel Mazar\u00e9","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mazar\u00e9, Pierre-Emmanuel","raw_affiliation_strings":["Meta FAIR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta FAIR","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136266429","display_name":"Lo\u00efc Cabannes","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cabannes, Lo\u00efc","raw_affiliation_strings":["Meta FAIR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta FAIR","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136211291","display_name":"Wen-tau Yih","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yih, Wen-tau","raw_affiliation_strings":["Meta FAIR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta FAIR","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111802678","display_name":"Herv\u00e9 Je\u01f5ou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"J\u00e9gou, Herv\u00e9","raw_affiliation_strings":["Meta FAIR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta FAIR","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3188000023365021,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3188000023365021,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.25450000166893005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.04639999940991402,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.6837999820709229},{"id":"https://openalex.org/keywords/memory-footprint","display_name":"Memory footprint","score":0.49410000443458557},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4797999858856201},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.4625999927520752},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.41920000314712524},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.4097000062465668},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.3522999882698059},{"id":"https://openalex.org/keywords/cpu-cache","display_name":"CPU cache","score":0.34459999203681946}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8327999711036682},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.6837999820709229},{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.49410000443458557},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4797999858856201},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4724000096321106},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4625999927520752},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.446399986743927},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.41920000314712524},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.4097000062465668},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3522999882698059},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.34459999203681946},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.33410000801086426},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.3100000023841858},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.2777999937534332},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.2754000127315521},{"id":"https://openalex.org/C2778565505","wikidata":"https://www.wikidata.org/wiki/Q2207566","display_name":"Spec#","level":2,"score":0.2612000107765198},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.25540000200271606},{"id":"https://openalex.org/C132943942","wikidata":"https://www.wikidata.org/wiki/Q2562511","display_name":"Footprint","level":2,"score":0.25360000133514404},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.2513999938964844}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.14037","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14037","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.14037","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14037","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.5272488594055176}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Under":[0],"modern":[1],"test-time":[2],"compute":[3],"and":[4,27,73,91,107,120,144,172,205],"agentic":[5],"paradigms,":[6],"language":[7],"models":[8],"process":[9],"ever-longer":[10],"sequences.":[11],"Efficient":[12],"text":[13],"generation":[14],"with":[15,175],"transformer":[16],"architectures":[17],"is":[18],"increasingly":[19],"constrained":[20],"by":[21,151],"the":[22,51,54,89,108,138,142,147,215],"Key-Value":[23,36],"cache":[24,90,149],"memory":[25,170],"footprint":[26],"bandwidth.":[28],"To":[29],"address":[30],"this":[31],"limitation,":[32],"we":[33,210],"introduce":[34],"Self-Pruned":[35],"Attention":[37],"(SP-KV),":[38],"a":[39,62,65,81,102,130,152,186],"mechanism":[40,139],"designed":[41],"to":[42,49,141,156,166,177,213],"predict":[43],"future":[44],"KV":[45,56,148],"utility":[46,67,100,109],"in":[47,88,93,169],"order":[48],"reduce":[50],"size":[52,150],"of":[53,154,180,189,217],"long-term":[55],"cache.":[57],"This":[58,164],"strategy":[59],"operates":[60],"at":[61],"fine":[63],"granularity:":[64],"lightweight":[66],"predictor":[68,110],"scores":[69],"each":[70],"key-value":[71],"pair,":[72],"while":[74],"recent":[75],"KVs":[76],"are":[77,86,111,121],"always":[78],"available":[79],"via":[80],"local":[82],"window,":[83],"older":[84],"pairs":[85],"written":[87],"used":[92],"global":[94],"attention":[95,220],"only":[96],"if":[97],"their":[98],"predicted":[99],"surpasses":[101],"given":[103],"threshold.":[104],"The":[105],"LLM":[106,125],"trained":[112],"jointly":[113],"end-to-end":[114],"exclusively":[115],"through":[116],"next-token":[117],"prediction":[118],"loss,":[119],"adapted":[122],"from":[123],"pretrained":[124],"checkpoints.":[126],"Rather":[127],"than":[128],"enforcing":[129],"fixed":[131],"compression":[132],"ratio,":[133],"SP-KV":[134],"performs":[135],"dynamic":[136],"sparsification:":[137],"adapts":[140],"input":[143],"typically":[145],"reduces":[146],"factor":[153],"$3$":[155],"$10\\times$,":[157],"longer":[158],"sequences":[159],"often":[160],"being":[161],"more":[162],"compressible.":[163],"leads":[165],"vast":[167],"improvements":[168],"usage":[171],"decoding":[173],"speed,":[174],"little":[176],"no":[178],"degradation":[179],"validation":[181],"loss":[182],"nor":[183],"performance":[184],"on":[185],"broad":[187],"set":[188],"downstream":[190],"tasks.":[191],"Beyond":[192],"serving":[193],"as":[194],"an":[195],"effective":[196],"KV-cache":[197],"reduction":[198],"mechanism,":[199],"our":[200],"method":[201],"reveals":[202],"structured":[203],"layer-":[204],"head-specific":[206],"sparsity":[207],"patterns":[208],"that":[209],"can":[211],"use":[212],"guide":[214],"design":[216],"hybrid":[218],"local-global":[219],"architectures.":[221]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-16T00:00:00"}