{"id":"https://openalex.org/W7160934808","doi":"https://doi.org/10.48550/arxiv.2605.09992","title":"Attention Drift: What Autoregressive Speculative Decoding Models Learn","display_name":"Attention Drift: What Autoregressive Speculative Decoding Models Learn","publication_year":2026,"publication_date":"2026-05-11","ids":{"openalex":"https://openalex.org/W7160934808","doi":"https://doi.org/10.48550/arxiv.2605.09992"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.09992","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09992","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.09992","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120004999","display_name":"Do\u011fa\u00e7 Eldenk","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Eldenk, Do\u011fa\u00e7","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113411377","display_name":"Payal Mohapatra","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mohapatra, Payal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052256497","display_name":"Yigitcan Comlek","orcid":"https://orcid.org/0000-0002-3654-1576"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Comlek, Yigitcan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009438864","display_name":"Kaan Oktay","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Oktay, Kaan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135941252","display_name":"Hongyang Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Hongyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5051730108","display_name":"Stephen Xia","orcid":"https://orcid.org/0000-0001-5713-8885"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Stephen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.5403000116348267,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.5403000116348267,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09359999746084213,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.09239999949932098,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/autoregressive-model","display_name":"Autoregressive model","score":0.6682000160217285},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.6304000020027161},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4893999993801117},{"id":"https://openalex.org/keywords/speculation","display_name":"Speculation","score":0.40549999475479126},{"id":"https://openalex.org/keywords/serialization","display_name":"Serialization","score":0.3962000012397766},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.3926999866962433},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.3889999985694885},{"id":"https://openalex.org/keywords/property","display_name":"Property (philosophy)","score":0.3391000032424927}],"concepts":[{"id":"https://openalex.org/C159877910","wikidata":"https://www.wikidata.org/wiki/Q2202883","display_name":"Autoregressive model","level":2,"score":0.6682000160217285},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6570000052452087},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.6304000020027161},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4893999993801117},{"id":"https://openalex.org/C47941915","wikidata":"https://www.wikidata.org/wiki/Q107885","display_name":"Speculation","level":2,"score":0.40549999475479126},{"id":"https://openalex.org/C52723943","wikidata":"https://www.wikidata.org/wiki/Q1127410","display_name":"Serialization","level":2,"score":0.3962000012397766},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.3926999866962433},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3889999985694885},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.34360000491142273},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3400000035762787},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.3391000032424927},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3319999873638153},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.3255000114440918},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.29589998722076416},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.2822999954223633},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.2773999869823456},{"id":"https://openalex.org/C197640229","wikidata":"https://www.wikidata.org/wiki/Q2534066","display_name":"Predictability","level":2,"score":0.26919999718666077},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.26820001006126404},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2678000032901764},{"id":"https://openalex.org/C2781020372","wikidata":"https://www.wikidata.org/wiki/Q533093","display_name":"On the fly","level":2,"score":0.26420000195503235},{"id":"https://openalex.org/C58468513","wikidata":"https://www.wikidata.org/wiki/Q1201165","display_name":"Determinacy","level":2,"score":0.2572000026702881},{"id":"https://openalex.org/C134652429","wikidata":"https://www.wikidata.org/wiki/Q1052698","display_name":"Jitter","level":2,"score":0.25029999017715454}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.09992","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09992","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.09992","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09992","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Speculative":[0],"decoding":[1],"accelerates":[2],"LLM":[3],"inference":[4],"by":[5,150],"drafting":[6,184],"future":[7],"tokens":[8,38],"with":[9,90,97],"a":[10,26,40,67,109],"small":[11],"model,":[12,147],"but":[13],"drafter":[14,35,70,127],"models":[15],"degrade":[16],"sharply":[17],"under":[18,154],"template":[19,155],"perturbation":[20],"and":[21,61,130,161,171],"long-context":[22,159],"inputs.":[23],"We":[24,54,72],"identify":[25],"previously-unreported":[27],"phenomenon":[28],"we":[29,119],"call":[30],"\\textbf{attention":[31],"drift}:":[32],"as":[33,108],"the":[34,47,76,83,104,117,126,144],"generates":[36],"successive":[37],"within":[39],"speculation":[41],"chain,":[42],"attention":[43],"progressively":[44],"moves":[45],"from":[46],"prompt":[48],"onto":[49],"its":[50],"own":[51],"recently-generated":[52],"tokens.":[53],"observe":[55],"this":[56,74],"across":[57],"both":[58],"\\emph{EAGLE3}":[59],"drafters":[60],"\\emph{MTP":[62],"heads},":[63],"suggesting":[64],"drift":[65],"is":[66],"property":[68],"of":[69],"designs.":[71],"trace":[73],"to":[75,115,152,180],"un-normalized":[77],"residual":[78],"path":[79],"between":[80],"chain":[81,91],"steps:":[82],"drafter's":[84],"hidden":[85,128,136],"state":[86],"magnitude":[87],"grows":[88],"monotonically":[89],"depth,":[92],"which":[93],"exhibits":[94],"dynamics":[95],"consistent":[96],"additional":[98],"pre-norm":[99,148],"transformer":[100],"layers":[101],"stacked":[102],"on":[103,125,158,163],"target":[105,135],"rather":[106],"than":[107],"standalone":[110],"autoregressive":[111],"predictor.":[112],"In":[113],"order":[114],"limit":[116],"growth,":[118],"propose":[120],"two":[121],"architectural":[122],"changes:":[123],"Post-norm":[124],"states":[129],"per-hidden-state":[131],"RMSNorm":[132],"after":[133],"capturing":[134],"states.":[137],"Our":[138,173],"interventions":[139],"improve":[140],"acceptance":[141],"length":[142],"over":[143,182],"current":[145],"leading":[146],"EAGLE3,":[149],"up":[151],"$2\\times$":[153],"perturbation,":[156],"$1.18\\times$":[157],"tasks,":[160],"$1.10\\times$":[162],"seven":[164],"standard":[165],"benchmarks":[166],"spanning":[167],"multi-turn":[168],"chat,":[169],"math,":[170],"coding.":[172],"changes":[174],"also":[175],"allow":[176],"shorter":[177],"train-time-test":[178],"depths":[179],"generalize":[181],"longer":[183],"sequences.":[185]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-13T00:00:00"}