{"id":"https://openalex.org/W7213612743","doi":"https://doi.org/10.48550/arxiv.2609.20734","title":"On-Demand Attention: Language Models Know When to Recall","display_name":"On-Demand Attention: Language Models Know When to Recall","publication_year":2026,"publication_date":"2026-09-17","ids":{"openalex":"https://openalex.org/W7213612743","doi":"https://doi.org/10.48550/arxiv.2609.20734"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.20734","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.20734","pdf_url":"https://arxiv.org/pdf/2609.20734","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.20734","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5152126191","display_name":"Haibo Feng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Feng, Haibo","raw_affiliation_strings":["Southern University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Southern University of Science and Technology","institution_ids":["https://openalex.org/I3045169105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129660687","display_name":"Ruiqi Liang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liang, Ruiqi","raw_affiliation_strings":["Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061118344","display_name":"Dongyang Jin","orcid":"https://orcid.org/0000-0002-5546-6655","observed_orcids":["https://orcid.org/0000-0002-5546-6655"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Dongyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153476227","display_name":"Hanyang Peng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peng, Hanyang","raw_affiliation_strings":["Peng Cheng Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peng Cheng Laboratory","institution_ids":["https://openalex.org/I4210136793"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5153520019","display_name":"Shiqi Yu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu, Shiqi","raw_affiliation_strings":["Southern University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Southern University of Science and Technology","institution_ids":["https://openalex.org/I3045169105"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5536999702453613,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5536999702453613,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4124000072479248,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.006300000008195639,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/long-context-inference","display_name":"long-context inference","score":0.7400000095367432},{"id":"https://openalex.org/keywords/international-attention","display_name":"global attention","score":0.8669999837875366},{"id":"https://openalex.org/keywords/local-attention","display_name":"local attention","score":0.6259999871253967},{"id":"https://openalex.org/keywords/kv-cache","display_name":"KV cache","score":0.5960000157356262},{"id":"https://openalex.org/keywords/hybrid-attention-mechanism","display_name":"hybrid attention mechanism","score":0.5839999914169312},{"id":"https://openalex.org/keywords/decoding-speedup","display_name":"decoding speedup","score":0.5070000290870667},{"id":"https://openalex.org/keywords/language-models","display_name":"language models","score":0.3529999852180481}],"concepts":[{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.7840999960899353},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.7415000200271606},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6588000059127808},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6144000291824341},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5946000218391418},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.4410000145435333},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.424699991941452},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.412200003862381},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4059999883174896},{"id":"https://openalex.org/C76679254","wikidata":"https://www.wikidata.org/wiki/Q5165163","display_name":"Context-dependent memory","level":4,"score":0.37040001153945923},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.364300012588501},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34040001034736633},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3402000069618225},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.3337000012397766},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.33230000734329224},{"id":"https://openalex.org/C36668950","wikidata":"https://www.wikidata.org/wiki/Q5500279","display_name":"Free recall","level":3,"score":0.3100999891757965}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.20734","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.20734","pdf_url":"https://arxiv.org/pdf/2609.20734","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.20734","doi":"https://doi.org/10.48550/arxiv.2609.20734","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.20734","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.20734","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.20734","pdf_url":"https://arxiv.org/pdf/2609.20734","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7213612743.pdf","grobid_xml":"https://content.openalex.org/works/W7213612743.grobid-xml"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W2046033161","https://openalex.org/W2626778328","https://openalex.org/W2908510526","https://openalex.org/W2979826702","https://openalex.org/W3006439205","https://openalex.org/W4386435778","https://openalex.org/W4387321091","https://openalex.org/W4389519019","https://openalex.org/W4389519118","https://openalex.org/W4389524599","https://openalex.org/W4394867990","https://openalex.org/W4402671258","https://openalex.org/W4402671766","https://openalex.org/W4407127938","https://openalex.org/W4411113142","https://openalex.org/W4412886690","https://openalex.org/W4412945131","https://openalex.org/W4414939355","https://openalex.org/W4415310891","https://openalex.org/W4417257988","https://openalex.org/W7114807665","https://openalex.org/W7117697722","https://openalex.org/W7138975341","https://openalex.org/W7143499254","https://openalex.org/W7152993773","https://openalex.org/W7165640989","https://openalex.org/W7166508455","https://openalex.org/W7167604932","https://openalex.org/W7204882795","https://openalex.org/W7207786570"],"related_works":["https://openalex.org/W7117697722","https://openalex.org/W7138975341","https://openalex.org/W7148411045","https://openalex.org/W4403571350","https://openalex.org/W7222308584","https://openalex.org/W7166645520","https://openalex.org/W7154151175","https://openalex.org/W4417536800","https://openalex.org/W7160600927","https://openalex.org/W7207786570"],"abstract_inverted_index":{"Reasoning":[0],"and":[1,101,108,131,191],"agentic":[2],"workloads":[3],"increasingly":[4],"demand":[5],"efficient":[6],"long-context":[7,158],"inference.":[8],"Yet":[9],"full-attention":[10],"decoding":[11,42,73,174,184],"reads":[12,171],"the":[13,20,38,41,86,95,110,143,152],"growing":[14],"history":[15],"at":[16,119],"every":[17],"step,":[18,40],"although":[19],"benefit":[21,58],"of":[22,56,142,154],"global":[23,33,90,155,170,201],"access":[24],"varies":[25],"across":[26,128],"prediction":[27,190],"positions.":[28],"We":[29],"find":[30],"that,":[31],"before":[32],"attention":[34,135,148],"is":[35],"computed":[36],"for":[37,124],"current":[39,87],"states":[43,185],"available":[44,123],"after":[45,75],"local":[46,60,76,147],"computation":[47,202],"in":[48,160],"frozen":[49],"pretrained":[50,105,183],"models":[51,198],"already":[52],"contain":[53],"information":[54,117],"predictive":[55],"its":[57],"over":[59,176],"attention.":[61,91,156,178],"Building":[62],"on":[63],"this":[64],"finding,":[65],"we":[66],"introduce":[67],"On-Demand":[68],"Attention":[69],"(ODA),":[70],"a":[71,78],"local-first":[72],"method:":[74],"computation,":[77],"lightweight":[79],"recall":[80,96],"head":[81,97],"decides":[82],"whether":[83],"to":[84,199],"recompute":[85],"step":[88,121],"with":[89,98],"ODA":[92,139],"trains":[93],"only":[94],"modest":[99],"data":[100],"compute":[102],"budgets,":[103],"leaving":[104],"weights":[106],"unchanged":[107],"retaining":[109],"complete":[111],"historical":[112],"KV":[113],"cache":[114],"so":[115],"that":[116,138,164,182],"skipped":[118],"one":[120],"remains":[122],"later":[125],"access.":[126],"Experiments":[127],"model":[129],"scales":[130],"families,":[132],"including":[133],"hybrid":[134],"backbones,":[136],"show":[137,163,181],"recovers":[140],"most":[141],"performance":[144],"lost":[145],"under":[146],"while":[149],"substantially":[150],"reducing":[151],"frequency":[153],"Controlled":[157],"measurements":[159],"vLLM":[161],"further":[162],"GPU-side":[165],"conditional":[166],"execution":[167],"translates":[168],"fewer":[169],"into":[172],"practical":[173],"speedups":[175],"full":[177],"These":[179],"findings":[180],"can":[186],"support":[187],"both":[188],"token":[189],"decisions":[192],"about":[193],"accessing":[194],"distant":[195],"information,":[196],"allowing":[197],"allocate":[200],"as":[203],"needed":[204],"during":[205],"decoding.":[206]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-09-19T00:00:00"}