{"id":"https://openalex.org/W7225002199","doi":"https://doi.org/10.5281/zenodo.23833121","title":"Attention Mechanisms: From Theory to Transformers","display_name":"Attention Mechanisms: From Theory to Transformers","publication_year":2026,"publication_date":"2026-10-31","ids":{"openalex":"https://openalex.org/W7225002199","doi":"https://doi.org/10.5281/zenodo.23833121"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.23833121","doi":"https://doi.org/10.5281/zenodo.23833121","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23833121","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.23833121","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5122728016","display_name":"Abhishek Pal","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I191972202","display_name":"Amity University","ror":"https://ror.org/02n9z0v62","country_code":"IN","type":"education","lineage":["https://openalex.org/I191972202"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"Pal, Abhishek","raw_affiliation_strings":["Amity School of Engineering and Technology, Amity University Uttar Pradesh, Lucknow, India"],"raw_orcid":"https://orcid.org/0009-0009-8842-1195","affiliations":[{"raw_affiliation_string":"Amity School of Engineering and Technology, Amity University Uttar Pradesh, Lucknow, India","institution_ids":["https://openalex.org/I191972202"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5122728016"],"corresponding_institution_ids":["https://openalex.org/I191972202"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8881999850273132,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8881999850273132,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0430000014603138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.014000000432133675,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/attention-mechanism","display_name":"attention mechanism","score":0.9990000128746033},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.7559999823570251},{"id":"https://openalex.org/keywords/scaled-dot-product-attention","display_name":"scaled dot-product attention","score":0.984000027179718},{"id":"https://openalex.org/keywords/multi-head-attention","display_name":"multi-head attention","score":0.996999979019165},{"id":"https://openalex.org/keywords/vanishing-gradient","display_name":"gradient vanishing","score":0.8460000157356262},{"id":"https://openalex.org/keywords/long-term-dependencies","display_name":"long-range dependencies","score":0.7559999823570251},{"id":"https://openalex.org/keywords/recurrent-neural-networks","display_name":"recurrent neural networks","score":0.8389999866485596},{"id":"https://openalex.org/keywords/long-short-term-memory","display_name":"long short-term memory","score":0.8830000162124634},{"id":"https://openalex.org/keywords/gradient-stability","display_name":"gradient stability","score":0.6449999809265137},{"id":"https://openalex.org/keywords/parallelization","display_name":"parallelization","score":0.492000013589859},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"natural language processing","score":0.7319999933242798},{"id":"https://openalex.org/keywords/computer-vision","display_name":"computer vision","score":0.9610000252723694},{"id":"https://openalex.org/keywords/speech-processing","display_name":"speech processing","score":0.6930000185966492}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.751800000667572},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6748999953269958},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5134999752044678},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.5013999938964844},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.48840001225471497},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4496999979019165},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.44350001215934753},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4242999851703644},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4196999967098236},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.3959999978542328},{"id":"https://openalex.org/C60008888","wikidata":"https://www.wikidata.org/wiki/Q6031013","display_name":"Information bottleneck method","level":3,"score":0.38769999146461487},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.38429999351501465},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3790000081062317},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.373199999332428},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.3481000065803528},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.33469998836517334},{"id":"https://openalex.org/C2778029271","wikidata":"https://www.wikidata.org/wiki/Q5421931","display_name":"Extension (predicate logic)","level":2,"score":0.32679998874664307},{"id":"https://openalex.org/C147168706","wikidata":"https://www.wikidata.org/wiki/Q1457734","display_name":"Recurrent neural network","level":3,"score":0.32659998536109924},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.3237999975681305},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3025999963283539},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.29910001158714294},{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.28780001401901245},{"id":"https://openalex.org/C155032097","wikidata":"https://www.wikidata.org/wiki/Q798503","display_name":"Backpropagation","level":3,"score":0.2759000062942505},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.27390000224113464},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.26739999651908875},{"id":"https://openalex.org/C12426560","wikidata":"https://www.wikidata.org/wiki/Q189569","display_name":"Basis (linear algebra)","level":2,"score":0.26420000195503235},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.2623000144958496},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.25850000977516174},{"id":"https://openalex.org/C52622258","wikidata":"https://www.wikidata.org/wiki/Q131222","display_name":"Information theory","level":2,"score":0.2578999996185303},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.2556999921798706},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.2526000142097473}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.23833121","doi":"https://doi.org/10.5281/zenodo.23833121","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23833121","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23833122","doi":"https://doi.org/10.5281/zenodo.23833122","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23833122","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.23833121","doi":"https://doi.org/10.5281/zenodo.23833121","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23833121","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3036475981","https://openalex.org/W7160695516","https://openalex.org/W7213354589","https://openalex.org/W7104719395","https://openalex.org/W7153953985","https://openalex.org/W7161838641","https://openalex.org/W7127918771","https://openalex.org/W7119527804","https://openalex.org/W2958545283","https://openalex.org/W7162202378"],"abstract_inverted_index":{"Recurrent":[0],"Neural":[1],"Networks":[2],"(RNNs)":[3],"and":[4,56,87,131,155,170,180,183,204,208,211],"Long":[5],"Short-Term":[6],"Memory":[7],"networks":[8],"(LSTMs)":[9],"have":[10],"a":[11,34,75,82,193],"well-documented":[12],"structural":[13,227],"limitation.":[14],"When":[15],"sequences":[16],"get":[17],"longer,":[18],"the":[19,29,70,94,136,143,147,201,226],"propagation":[20],"of":[21,68,139,195],"gradient":[22,31,171],"signals":[23],"becomes":[24],"increasingly":[25],"unreliable,":[26],"known":[27],"as":[28,51],"vanishing":[30],"problem.":[32],"Such":[33],"weakness":[35],"naturally":[36],"limits":[37],"their":[38],"ability":[39],"to":[40,48,84,152,159],"model":[41,83],"long-range":[42],"token":[43,114],"dependencies,":[44],"which":[45],"are":[46,216],"crucial":[47],"tasks":[49],"such":[50],"machine":[52],"translation,":[53],"document":[54],"summarisation,":[55],"question":[57],"answering.":[58],"This":[59],"bottleneck":[60],"is":[61,192],"directly":[62],"addressed":[63],"by":[64],"attention":[65,80,127],"mechanisms.":[66],"Instead":[67],"encoding":[69],"whole":[71],"input":[72,95],"sequence":[73,107],"into":[74],"single":[76],"fixed-length":[77],"hidden":[78],"state,":[79],"enables":[81],"dynamically":[85],"retrieve":[86],"weight":[88],"information":[89,167],"from":[90,129],"any":[91],"position":[92],"in":[93],"at":[96,120],"each":[97],"decoding":[98],"step.":[99],"The":[100,190],"resulting":[101],"computational":[102,165],"cost":[103],"scales":[104],"quadratically":[105],"with":[106,219,224],"length":[108,169],"but":[109,223],"allows":[110],"full":[111],"parallelisation":[112],"across":[113],"positions,":[115],"making":[116],"attention-based":[117,184],"architectures":[118,161],"tractable":[119],"scale.":[121],"In":[122],"this":[123],"survey":[124],"we":[125],"investigate":[126],"mechanisms":[128],"mathematical":[130],"empirical":[132,221],"perspectives.":[133],"We":[134,215],"analyse":[135],"theoretical":[137],"basis":[138],"scaled":[140],"dot-product":[141],"attention,":[142,154],"statistical":[144],"motivation":[145],"for":[146,229],"scaling":[148],"factor,":[149],"its":[150],"extension":[151],"multi-head":[153],"formally":[156],"compare":[157],"it":[158],"recurrent":[160],"along":[162],"three":[163],"axes:":[164],"parallelism,":[166],"path":[168],"stability.":[172],"It":[173],"covers":[174],"natural":[175],"language":[176],"processing,":[177],"computer":[178],"vision":[179],"speech":[181],"systems,":[182],"models":[185],"always":[186],"beat":[187],"sequential":[188],"models.":[189],"methodology":[191],"blend":[194],"systematic":[196],"literature":[197],"review,":[198],"based":[199],"on":[200],"ACL":[202],"Anthology":[203],"arXiv":[205],"(cs.LG,":[206],"2014-2024),":[207],"formal":[209],"derivations":[210],"illustrative":[212],"numerical":[213],"analysis.":[214],"not":[217],"concerned":[218],"listing":[220],"findings,":[222],"showing":[225],"reasons":[228],"them.":[230]},"counts_by_year":[],"updated_date":"2026-11-02T09:45:03.842371","created_date":"2026-11-01T00:00:00"}