{"id":"https://openalex.org/W3132023005","doi":null,"title":"Memory Representation in Transformer","display_name":"Memory Representation in Transformer","publication_year":2021,"publication_date":"2021-05-04","ids":{"openalex":"https://openalex.org/W3132023005","mag":"3132023005"},"language":"en","primary_location":{"id":"mag:3132023005","doi":null,"is_oa":false,"landing_page_url":"https://openreview.net/pdf?id=g1KmTQhOhag","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065291780","display_name":"Mikhail Burtsev","orcid":"https://orcid.org/0000-0003-1614-1695","observed_orcids":["https://orcid.org/0000-0003-1614-1695"]},"institutions":[{"id":"https://openalex.org/I153845743","display_name":"Moscow Institute of Physics and Technology","ror":"https://ror.org/00v0z9322","country_code":"RU","type":"education","lineage":["https://openalex.org/I153845743"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Mikhail Burtsev","raw_affiliation_strings":["Moscow Institute of Physics and Technology"],"raw_orcid":"https://orcid.org/0000-0003-1614-1695","affiliations":[{"raw_affiliation_string":"Moscow Institute of Physics and Technology","institution_ids":["https://openalex.org/I153845743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032492419","display_name":"Yuri Kuratov","orcid":"https://orcid.org/0009-0007-0718-0824","observed_orcids":["https://orcid.org/0009-0007-0718-0824"]},"institutions":[{"id":"https://openalex.org/I153845743","display_name":"Moscow Institute of Physics and Technology","ror":"https://ror.org/00v0z9322","country_code":"RU","type":"education","lineage":["https://openalex.org/I153845743"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Yuri Kuratov","raw_affiliation_strings":["Moscow Institute of Physics and Technology"],"raw_orcid":"https://orcid.org/0009-0007-0718-0824","affiliations":[{"raw_affiliation_string":"Moscow Institute of Physics and Technology","institution_ids":["https://openalex.org/I153845743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037775510","display_name":"Anton Peganov","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1304085615","display_name":"Nvidia (United Kingdom)","ror":"https://ror.org/02kr42612","country_code":"GB","type":"company","lineage":["https://openalex.org/I1304085615","https://openalex.org/I4210127875"]},{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Anton Peganov","raw_affiliation_strings":["nVidia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"nVidia","institution_ids":["https://openalex.org/I1304085615","https://openalex.org/I4210127875"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5073759343","display_name":"Grigory V. Sapunov","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Grigory V. Sapunov","raw_affiliation_strings":["Intento"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Intento","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.02056679,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9506999850273132,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9506999850273132,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.03420000150799751,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.003100000089034438,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/memory-augmented-neural-networks","display_name":"memory-augmented neural networks","score":0.8790000081062317},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.5379999876022339},{"id":"https://openalex.org/keywords/self-attention","display_name":"self-attention","score":0.7599999904632568},{"id":"https://openalex.org/keywords/memory-bottleneck","display_name":"memory bottleneck","score":0.8169999718666077},{"id":"https://openalex.org/keywords/machine-translation","display_name":"machine translation","score":0.871999979019165},{"id":"https://openalex.org/keywords/language-modeling","display_name":"language modeling","score":0.8560000061988831},{"id":"https://openalex.org/keywords/attention-visualization","display_name":"attention visualization","score":0.6240000128746033},{"id":"https://openalex.org/keywords/glue-benchmark","display_name":"GLUE benchmark","score":0.6179999709129333},{"id":"https://openalex.org/keywords/masked-language-models","display_name":"masked language models","score":0.7120000123977661}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8435547351837158},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6927436590194702},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.6051007509231567},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.4833640456199646},{"id":"https://openalex.org/C82687282","wikidata":"https://www.wikidata.org/wiki/Q66221","display_name":"Auxiliary memory","level":2,"score":0.47732076048851013},{"id":"https://openalex.org/C147168706","wikidata":"https://www.wikidata.org/wiki/Q1457734","display_name":"Recurrent neural network","level":3,"score":0.4759601950645447},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45683544874191284},{"id":"https://openalex.org/C12186640","wikidata":"https://www.wikidata.org/wiki/Q6815743","display_name":"Memory model","level":3,"score":0.43235501646995544},{"id":"https://openalex.org/C74426580","wikidata":"https://www.wikidata.org/wiki/Q719484","display_name":"Memory map","level":3,"score":0.42535579204559326},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.39363789558410645},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.39052441716194153},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.15942701697349548},{"id":"https://openalex.org/C133875982","wikidata":"https://www.wikidata.org/wiki/Q764810","display_name":"Shared memory","level":2,"score":0.13377779722213745},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.08994784951210022},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:3132023005","doi":null,"is_oa":false,"landing_page_url":"https://openreview.net/pdf?id=g1KmTQhOhag","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4394667418","https://openalex.org/W7133228637","https://openalex.org/W4303449058","https://openalex.org/W3033182847","https://openalex.org/W3215473765","https://openalex.org/W4404783458","https://openalex.org/W4306751627","https://openalex.org/W4414458022","https://openalex.org/W2955227499","https://openalex.org/W4403579218"],"abstract_inverted_index":{"Transformer-based":[0],"models":[1],"have":[2,96],"achieved":[3],"state-of-the-art":[4],"results":[5,204],"in":[6,37],"many":[7],"natural":[8],"language":[9,122,190,197],"processing":[10,46],"tasks.":[11,192],"The":[12],"self-attention":[13],"architecture":[14],"allows":[15],"transformer":[16],"to":[17,50,61,77,100,121,149,224],"combine":[18],"information":[19,30],"from":[20,118,207],"all":[21],"elements":[22],"of":[23,47,70,128,140,178,194,211],"a":[24,54,71,74,226],"sequence":[25,52,72],"into":[26],"context-aware":[27],"representations.":[28,41,94],"However,":[29],"about":[31],"the":[32,38,45,51,79,98,141,158,183,215,221],"context":[33],"is":[34,73],"stored":[35],"mostly":[36],"same":[39],"element-wise":[40],"This":[42],"might":[43],"limit":[44],"properties":[48],"related":[49],"as":[53,65,67],"whole":[55],"more":[56],"difficult.":[57],"Adding":[58],"trainable":[59],"memory":[60,92,147,155,163,171,179,200,216],"selectively":[62],"store":[63,150],"local":[64],"well":[66],"global":[68,159,227],"representations":[69],"promising":[75],"direction":[76],"improve":[78],"Transformer":[80,142],"model.":[81],"Memory-augmented":[82],"neural":[83,88],"networks":[84],"(MANNs)":[85],"extend":[86],"traditional":[87],"architectures":[89],"with":[90,165,182,199],"general-purpose":[91],"for":[93,157,186,205],"MANNs":[95],"demonstrated":[97],"capability":[99],"learn":[101],"simple":[102],"algorithms":[103],"like":[104],"Copy":[105],"or":[106],"Reverse":[107],"and":[108,126,136,174,189],"can":[109],"be":[110],"successfully":[111],"trained":[112],"via":[113],"backpropagation":[114],"on":[115],"diverse":[116],"tasks":[117,206],"question":[119],"answering":[120],"modeling":[123],"outperforming":[124],"RNNs":[125],"LSTMs":[127],"comparable":[129],"complexity.":[130],"In":[131],"this":[132],"work,":[133],"we":[134],"propose":[135],"study":[137],"few":[138],"extensions":[139],"baseline":[143],"(1)":[144],"by":[145],"adding":[146],"tokens":[148,201],"non-local":[151],"representations,":[152],"(2)":[153],"creating":[154],"bottleneck":[156],"information,":[160],"(3)":[161],"controlling":[162],"update":[164],"dedicated":[166],"layer.":[167],"We":[168],"evaluate":[169],"these":[170],"augmented":[172],"Transformers":[173],"demonstrate":[175],"that":[176,218],"presence":[177],"positively":[180],"correlates":[181],"model":[184,198],"performance":[185],"machine":[187],"translation":[188],"modelling":[191],"Augmentation":[193],"pre-trained":[195],"masked":[196],"shows":[202],"mixed":[203],"GLUE":[208],"benchmark.":[209],"Visualization":[210],"attention":[212],"patterns":[213],"over":[214],"suggest":[217],"it":[219],"improves":[220],"model's":[222],"ability":[223],"process":[225],"context.":[228]},"counts_by_year":[],"updated_date":"2025-11-06T04:12:42.849631","created_date":"2021-03-01T00:00:00"}