{"id":"https://openalex.org/W7162071591","doi":"https://doi.org/10.48550/arxiv.2605.22791","title":"Gated DeltaNet-2: Decoupling Erase and Write in Linear Attention","display_name":"Gated DeltaNet-2: Decoupling Erase and Write in Linear Attention","publication_year":2026,"publication_date":"2026-05-21","ids":{"openalex":"https://openalex.org/W7162071591","doi":"https://doi.org/10.48550/arxiv.2605.22791"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.22791","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22791","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.22791","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028525720","display_name":"Ali Hatamizadeh","orcid":"https://orcid.org/0000-0002-5349-1996"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hatamizadeh, Ali","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136753813","display_name":"Yejin Choi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Choi, Yejin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136781474","display_name":"Jan Kautz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kautz, Jan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5424000024795532,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5424000024795532,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.15219999849796295,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.07280000299215317,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.49129998683929443},{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.4018000066280365},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.3813999891281128},{"id":"https://openalex.org/keywords/scalar","display_name":"Scalar (mathematics)","score":0.37860000133514404},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.36880001425743103},{"id":"https://openalex.org/keywords/decoupling","display_name":"Decoupling (probability)","score":0.3587000072002411},{"id":"https://openalex.org/keywords/polarity","display_name":"Polarity (international relations)","score":0.35350000858306885},{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.3474999964237213}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6283000111579895},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.49570000171661377},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.49129998683929443},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.4018000066280365},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.3813999891281128},{"id":"https://openalex.org/C57691317","wikidata":"https://www.wikidata.org/wiki/Q1289248","display_name":"Scalar (mathematics)","level":2,"score":0.37860000133514404},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.36880001425743103},{"id":"https://openalex.org/C205606062","wikidata":"https://www.wikidata.org/wiki/Q5249645","display_name":"Decoupling (probability)","level":2,"score":0.3587000072002411},{"id":"https://openalex.org/C2777361361","wikidata":"https://www.wikidata.org/wiki/Q1112585","display_name":"Polarity (international relations)","level":3,"score":0.35350000858306885},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.3474999964237213},{"id":"https://openalex.org/C172385210","wikidata":"https://www.wikidata.org/wiki/Q5339","display_name":"Transistor","level":3,"score":0.31869998574256897},{"id":"https://openalex.org/C182548165","wikidata":"https://www.wikidata.org/wiki/Q2261483","display_name":"Scrambling","level":2,"score":0.3156000077724457},{"id":"https://openalex.org/C138777275","wikidata":"https://www.wikidata.org/wiki/Q6884054","display_name":"Mixing (physics)","level":2,"score":0.30630001425743103},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3052000105381012},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.2994000017642975},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.2849999964237213},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.2777000069618225},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.27459999918937683},{"id":"https://openalex.org/C184720557","wikidata":"https://www.wikidata.org/wiki/Q7825049","display_name":"Topology (electrical circuits)","level":2,"score":0.2605000138282776},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.2590000033378601},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.25780001282691956},{"id":"https://openalex.org/C311688","wikidata":"https://www.wikidata.org/wiki/Q2393193","display_name":"Time complexity","level":2,"score":0.25699999928474426},{"id":"https://openalex.org/C2776141515","wikidata":"https://www.wikidata.org/wiki/Q1274479","display_name":"Repetition (rhetorical device)","level":2,"score":0.25099998712539673},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.25029999017715454}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.22791","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22791","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.22791","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22791","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.5645900368690491,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Linear":[0],"attention":[1,8],"replaces":[2],"the":[3,48,67,88,99,125,158,166,209,241],"unbounded":[4],"cache":[5],"of":[6],"softmax":[7],"with":[9,63,138,180],"a":[10,53,72,139,145,172,176,189],"fixed-size":[11],"recurrent":[12,251],"state,":[13],"reducing":[14,150],"sequence":[15],"mixing":[16],"to":[17,22,32,36,76,85,96,151,157,162],"linear":[18],"time":[19],"and":[20,56,91,111,117,130,144,161,188,218,226,246,252],"decoding":[21],"constant":[23],"memory.":[24],"The":[25],"hard":[26],"part":[27],"is":[28,230,256],"not":[29],"just":[30],"what":[31],"forget,":[33],"but":[34],"how":[35,81,92],"edit":[37,69],"this":[38],"compressed":[39],"memory":[40],"without":[41],"scrambling":[42],"existing":[43],"associations.":[44],"Delta-rule":[45],"models":[46],"subtract":[47],"current":[49],"read":[50],"before":[51],"writing":[52],"new":[54,94],"value,":[55],"Kimi":[57],"Delta":[58,133],"Attention":[59],"(KDA)":[60],"sharpens":[61],"forgetting":[62,116],"channel-wise":[64,118,140,146,181],"decay.":[65],"But":[66],"active":[68],"still":[70],"uses":[71],"single":[73],"scalar":[74,126,160],"gate":[75,142,148],"control":[77],"two":[78],"different":[79],"things:":[80],"much":[82,93],"old":[83],"content":[84,95],"erase":[86,141,186],"on":[87,98,202,233],"key":[89],"side":[90],"commit":[97],"value":[100],"side.":[101],"We":[102,170],"introduce":[103],"Gated":[104,109,132,163,206,215],"DeltaNet-2,":[105],"which":[106],"generalizes":[107],"both":[108,154,250],"DeltaNet":[110,164],"KDA":[112,152],"by":[113],"inheriting":[114],"adaptive":[115],"decay":[119,167,182],"while":[120],"addressing":[121],"their":[122],"shared":[123],"limitation,":[124],"tie":[127],"between":[128],"erasing":[129],"writing.":[131],"Rule-2":[134],"separates":[135],"these":[136],"roles":[137],"b_t":[143],"write":[147],"w_t,":[149],"when":[153,165],"gates":[155],"collapse":[156],"same":[159],"also":[168],"collapses.":[169],"derive":[171],"fast-weight":[173],"update":[174],"view,":[175],"chunkwise":[177],"WY":[178],"algorithm":[179],"absorbed":[183],"into":[184],"asymmetric":[185],"factors,":[187],"gate-aware":[190],"backward":[191],"pass":[192],"that":[193],"preserves":[194],"efficient":[195],"parallel":[196],"training.":[197],"At":[198],"1.3B":[199],"parameters":[200],"trained":[201],"100B":[203],"FineWeb-Edu":[204],"tokens,":[205],"DeltaNet-2":[207],"achieves":[208],"strongest":[210],"overall":[211],"results":[212],"among":[213],"Mamba-2,":[214],"DeltaNet,":[216],"KDA,":[217],"Mamba-3":[219],"variants":[220],"across":[221],"language":[222],"modeling,":[223],"commonsense":[224],"reasoning,":[225],"retrieval.":[227],"Its":[228],"advantage":[229],"most":[231],"pronounced":[232],"long-context":[234],"RULER":[235],"needle-in-a-haystack":[236],"benchmarks,":[237],"where":[238],"it":[239],"improves":[240],"evaluated":[242],"multi-key":[243],"retrieval":[244],"setting":[245],"remains":[247],"strong":[248],"in":[249],"hybrid":[253],"settings.":[254],"Code":[255],"available":[257],"at":[258],"https://github.com/NVlabs/GatedDeltaNet-2.":[259]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-23T00:00:00"}