{"id":"https://openalex.org/W7164179495","doi":"https://doi.org/10.48550/arxiv.2606.11052","title":"Attention Amnesia in Hybrid LLMs: When CoT Fine-Tuning Breaks Long-Range Recall, and How to Fix It","display_name":"Attention Amnesia in Hybrid LLMs: When CoT Fine-Tuning Breaks Long-Range Recall, and How to Fix It","publication_year":2026,"publication_date":"2026-06-09","ids":{"openalex":"https://openalex.org/W7164179495","doi":"https://doi.org/10.48550/arxiv.2606.11052"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.11052","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.11052","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.11052","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138344201","display_name":"Xinyu Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Xinyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138318943","display_name":"Boyu Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Boyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138328842","display_name":"Yi Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Yi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138363751","display_name":"Zhiwei Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Zhiwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138355118","display_name":"Yingfa Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yingfa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138389010","display_name":"Huiming Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Huiming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138333699","display_name":"Zhijiang Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Zhijiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.10480000078678131,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.10480000078678131,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.10189999639987946,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.09679999947547913,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6416000127792358},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.5708000063896179},{"id":"https://openalex.org/keywords/anterograde-amnesia","display_name":"Anterograde amnesia","score":0.37380000948905945},{"id":"https://openalex.org/keywords/amnesia","display_name":"Amnesia","score":0.3612000048160553},{"id":"https://openalex.org/keywords/debiasing","display_name":"Debiasing","score":0.33629998564720154}],"concepts":[{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6416000127792358},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5841000080108643},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.5708000063896179},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46970000863075256},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.4530999958515167},{"id":"https://openalex.org/C2775847465","wikidata":"https://www.wikidata.org/wiki/Q572111","display_name":"Anterograde amnesia","level":3,"score":0.37380000948905945},{"id":"https://openalex.org/C2778615406","wikidata":"https://www.wikidata.org/wiki/Q11072","display_name":"Amnesia","level":2,"score":0.3612000048160553},{"id":"https://openalex.org/C2779458634","wikidata":"https://www.wikidata.org/wiki/Q24963715","display_name":"Debiasing","level":2,"score":0.33629998564720154},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33550000190734863},{"id":"https://openalex.org/C168031717","wikidata":"https://www.wikidata.org/wiki/Q1530280","display_name":"Balance (ability)","level":2,"score":0.32829999923706055},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3280999958515167},{"id":"https://openalex.org/C76679254","wikidata":"https://www.wikidata.org/wiki/Q5165163","display_name":"Context-dependent memory","level":4,"score":0.2653000056743622},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.26170000433921814}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.11052","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.11052","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.11052","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.11052","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Chain-of-thought":[0],"(CoT)":[1],"supervised":[2],"fine-tuning":[3],"(SFT)":[4],"is":[5],"widely":[6],"adopted":[7],"to":[8,62,67,118,149],"improve":[9],"reasoning":[10,123,138,154],"ability,":[11],"yet":[12],"we":[13,90],"find":[14],"that":[15,80,96],"it":[16,144],"systematically":[17],"degrades":[18],"long-context":[19,130],"recall":[20],"in":[21],"hybrid":[22],"linear-attention":[23],"models.":[24],"Across":[25,125],"architectures":[26],"including":[27],"HypeNet":[28],"and":[29,40,50,100,122],"Jet-Nemotron,":[30],"retrieval":[31,48],"performance":[32],"on":[33,57,142],"Needle-In-A-Haystack":[34],"(NIAH)":[35],"deteriorates":[36],"substantially":[37],"after":[38],"CoT-SFT,":[39],"the":[41,103],"degradation":[42],"becomes":[43],"more":[44],"severe":[45],"under":[46],"harder":[47],"settings":[49],"longer":[51],"context":[52],"windows.":[53],"For":[54],"example,":[55],"HypeNet-9B":[56],"NIAH-S2@256K":[58],"decreases":[59],"from":[60,102,147],"$67.2\\%$":[61],"$9.4\\%$.":[63],"We":[64,112],"attribute":[65],"this":[66,88],"CoT-SFT":[68],"biasing":[69],"attention":[70],"gradients":[71],"toward":[72],"short-range":[73],"patterns,":[74],"disrupting":[75],"query-key":[76],"projections":[77],"($W_Q,":[78],"W_K$)":[79],"are":[81],"responsible":[82],"for":[83,140],"long-range":[84],"routing.":[85],"Motivated":[86],"by":[87],"observation,":[89],"propose":[91],"QK-Restore,":[92],"a":[93,115],"training-free":[94],"method":[95],"restores":[97,129],"only":[98],"$W_Q$":[99],"$W_K$":[101],"pre-SFT":[104],"checkpoint":[105],"while":[106,136,151],"preserving":[107,137],"all":[108],"other":[109],"post-SFT":[110],"parameters.":[111],"further":[113],"introduce":[114],"Procrustes":[116],"variant":[117],"balance":[119],"routing":[120],"preservation":[121],"adaptation.":[124],"architectures,":[126],"QK-Restore":[127],"consistently":[128],"capability":[131],"at":[132],"zero":[133],"training":[134],"cost":[135],"performance;":[139],"instance,":[141],"HypeNet-5B":[143],"improves":[145],"S3@256K":[146],"$65.4\\%$":[148],"$76.4\\%$":[150],"maintaining":[152],"strong":[153],"performance.":[155]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-11T00:00:00"}