{"id":"https://openalex.org/W2952744660","doi":"https://doi.org/10.1162/tacl_a_00306","title":"Theoretical Limitations of Self-Attention in Neural Sequence Models","display_name":"Theoretical Limitations of Self-Attention in Neural Sequence Models","publication_year":2020,"publication_date":"2020-03-31","ids":{"openalex":"https://openalex.org/W2952744660","doi":"https://doi.org/10.1162/tacl_a_00306","mag":"2952744660"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00306","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00306","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00306/1923102/tacl_a_00306.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00306/1923102/tacl_a_00306.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007794565","display_name":"Michael Hahn","orcid":"https://orcid.org/0000-0003-4828-4834"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Michael Hahn","raw_affiliation_strings":["Stanford University","Stanford University, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Stanford University, United States","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5007794565"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":9.6514,"has_fulltext":true,"cited_by_count":143,"citation_normalized_percentile":{"value":0.9842528,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"8","issue":null,"first_page":"156","last_page":"171"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9930999875068665,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8162616491317749},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.642929196357727},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4769101142883301},{"id":"https://openalex.org/keywords/computational-model","display_name":"Computational model","score":0.47520431876182556},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.46185505390167236},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.4496026635169983},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4400680959224701},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.4275190234184265},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4211200475692749},{"id":"https://openalex.org/keywords/computational-linguistics","display_name":"Computational linguistics","score":0.42005494236946106},{"id":"https://openalex.org/keywords/formal-language","display_name":"Formal language","score":0.41738373041152954},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.39176449179649353},{"id":"https://openalex.org/keywords/cognitive-science","display_name":"Cognitive science","score":0.33987343311309814},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.22455772757530212},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.07986050844192505}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8162616491317749},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.642929196357727},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4769101142883301},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.47520431876182556},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.46185505390167236},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.4496026635169983},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4400680959224701},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.4275190234184265},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4211200475692749},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.42005494236946106},{"id":"https://openalex.org/C146072743","wikidata":"https://www.wikidata.org/wiki/Q192161","display_name":"Formal language","level":2,"score":0.41738373041152954},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.39176449179649353},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.33987343311309814},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.22455772757530212},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.07986050844192505},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1162/tacl_a_00306","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00306","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00306/1923102/tacl_a_00306.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1906.06755","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1906.06755","pdf_url":"https://arxiv.org/pdf/1906.06755","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:doaj.org/article:9a179321a285476ca44f605eba696e3e","is_oa":true,"landing_page_url":"https://doaj.org/article/9a179321a285476ca44f605eba696e3e","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 8, Pp 156-171 (2020)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00306","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00306","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00306/1923102/tacl_a_00306.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.8500000238418579}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2952744660.pdf","grobid_xml":"https://content.openalex.org/works/W2952744660.grobid-xml"},"referenced_works_count":87,"referenced_works":["https://openalex.org/W137331784","https://openalex.org/W1527197079","https://openalex.org/W1585040575","https://openalex.org/W1741580681","https://openalex.org/W1890888469","https://openalex.org/W1900614497","https://openalex.org/W1972594149","https://openalex.org/W2006991246","https://openalex.org/W2045924252","https://openalex.org/W2060270693","https://openalex.org/W2067619114","https://openalex.org/W2077439714","https://openalex.org/W2087807962","https://openalex.org/W2088559598","https://openalex.org/W2107265154","https://openalex.org/W2107878631","https://openalex.org/W2121029939","https://openalex.org/W2125763748","https://openalex.org/W2144836808","https://openalex.org/W2158191188","https://openalex.org/W2160732323","https://openalex.org/W2164548001","https://openalex.org/W2190736972","https://openalex.org/W2267186426","https://openalex.org/W2406655293","https://openalex.org/W2413794162","https://openalex.org/W2549835527","https://openalex.org/W2597655663","https://openalex.org/W2608651664","https://openalex.org/W2612675303","https://openalex.org/W2612690371","https://openalex.org/W2798727047","https://openalex.org/W2866343820","https://openalex.org/W2884991722","https://openalex.org/W2888922637","https://openalex.org/W2896060389","https://openalex.org/W2896457183","https://openalex.org/W2898226569","https://openalex.org/W2899702797","https://openalex.org/W2905889633","https://openalex.org/W2906125418","https://openalex.org/W2908802752","https://openalex.org/W2914557243","https://openalex.org/W2918871811","https://openalex.org/W2940744433","https://openalex.org/W2946417913","https://openalex.org/W2946794439","https://openalex.org/W2947469743","https://openalex.org/W2948981535","https://openalex.org/W2949292434","https://openalex.org/W2949674892","https://openalex.org/W2951977278","https://openalex.org/W2953307569","https://openalex.org/W2962729168","https://openalex.org/W2962749806","https://openalex.org/W2962911926","https://openalex.org/W2963059228","https://openalex.org/W2963341956","https://openalex.org/W2963403868","https://openalex.org/W2963575866","https://openalex.org/W2963641307","https://openalex.org/W2963723151","https://openalex.org/W2963751529","https://openalex.org/W2963753324","https://openalex.org/W2963768805","https://openalex.org/W2964045208","https://openalex.org/W2964110616","https://openalex.org/W2964189376","https://openalex.org/W2972324944","https://openalex.org/W2972515356","https://openalex.org/W2973154008","https://openalex.org/W2981584912","https://openalex.org/W2983965200","https://openalex.org/W3037001987","https://openalex.org/W3099624838","https://openalex.org/W3142632021","https://openalex.org/W4229781645","https://openalex.org/W4235611032","https://openalex.org/W4238634189","https://openalex.org/W4238977241","https://openalex.org/W4240217567","https://openalex.org/W4245255589","https://openalex.org/W4250083867","https://openalex.org/W4288328877","https://openalex.org/W4289306505","https://openalex.org/W4323654151","https://openalex.org/W4385245566"],"related_works":["https://openalex.org/W2169518243","https://openalex.org/W197753654","https://openalex.org/W284959975","https://openalex.org/W2252095989","https://openalex.org/W2360038484","https://openalex.org/W2105076537","https://openalex.org/W1557764619","https://openalex.org/W2153406043","https://openalex.org/W1500595708","https://openalex.org/W1589891421"],"abstract_inverted_index":{"Transformers":[0],"are":[1,37,124],"emerging":[2],"as":[3],"the":[4,28,45,66,83,98,104,128],"new":[5],"workhorse":[6],"of":[7,31,48,65,69,85,101],"NLP,":[8],"showing":[9],"great":[10],"success":[11,100],"across":[12],"tasks.":[13],"Unlike":[14],"LSTMs,":[15],"transformers":[16],"process":[17,34],"input":[18,91],"sequences":[19],"entirely":[20],"through":[21],"self-attention.":[22],"Previous":[23],"work":[24],"has":[25],"suggested":[26],"that":[27,72,114,123],"computational":[29,46,67],"capabilities":[30],"self-attention":[32,49,102],"to":[33,50,108],"hierarchical":[35,80,109],"structures":[36],"limited.":[38],"In":[39],"this":[40],"work,":[41],"we":[42,60],"mathematically":[43],"investigate":[44],"power":[47],"model":[51,75],"formal":[52,129],"languages.":[53],"Across":[54],"both":[55],"soft":[56],"and":[57,103],"hard":[58],"attention,":[59],"show":[61],"strong":[62],"theoretical":[63,134],"limitations":[64,94],"abilities":[68],"self-attention,":[70],"finding":[71],"it":[73],"cannot":[74],"periodic":[76],"finite-state":[77],"languages,":[78],"nor":[79],"structure,":[81],"unless":[82],"number":[84],"layers":[86],"or":[87],"heads":[88],"increases":[89],"with":[90,121],"length.":[92],"These":[93],"seem":[95],"surprising":[96],"given":[97],"practical":[99],"prominent":[105],"role":[106],"assigned":[107],"structure":[110],"in":[111,133],"linguistics,":[112],"suggesting":[113],"natural":[115],"language":[116],"can":[117],"be":[118],"approximated":[119],"well":[120],"models":[122],"too":[125],"weak":[126],"for":[127],"languages":[130],"typically":[131],"assumed":[132],"linguistics.":[135]},"counts_by_year":[{"year":2026,"cited_by_count":18},{"year":2025,"cited_by_count":28},{"year":2024,"cited_by_count":24},{"year":2023,"cited_by_count":16},{"year":2022,"cited_by_count":10},{"year":2021,"cited_by_count":29},{"year":2020,"cited_by_count":17},{"year":2019,"cited_by_count":1}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}