{"id":"https://openalex.org/W7133196742","doi":"https://doi.org/10.52202/075280-2755","title":"Fast Attention Requires Bounded Entries","display_name":"Fast Attention Requires Bounded Entries","alternate_titles":[],"publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W7133196742","doi":"https://doi.org/10.52202/075280-2755"},"language":"en","primary_location":{"id":"doi:10.52202/075280-2755","doi":"https://doi.org/10.52202/075280-2755","is_oa":false,"landing_page_url":"https://doi.org/10.52202/075280-2755","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":"https://openalex.org/P4404678063","host_organization_name":"Neural Information Processing Systems Foundation","host_organization_lineage":["https://openalex.org/P4404678063"],"host_organization_lineage_names":["Neural Information Processing Systems Foundation"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 36","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W7133196742"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2302.13214","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006981411","display_name":"Josh Alman","orcid":"https://orcid.org/0009-0002-2204-1359","observed_orcids":["https://orcid.org/0009-0002-2204-1359"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Josh Alman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5008391481","display_name":"Zhao Song","orcid":"https://orcid.org/0000-0002-0897-6724","observed_orcids":["https://orcid.org/0000-0002-0897-6724"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao Song","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":14.327,"has_fulltext":true,"cited_by_count":138,"citation_normalized_percentile":{"value":0.99658703,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"63117","last_page":"63135"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10720","display_name":"Complexity and Algorithms in Graphs","score":0.9286999702453613,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10720","display_name":"Complexity and Algorithms in Graphs","score":0.9286999702453613,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.007000000216066837,"subfield":{"id":"https://openalex.org/subfields/2614","display_name":"Theoretical Computer Science"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10792","display_name":"Matrix Theory and Algorithms","score":0.004900000058114529,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/attention-computation","display_name":"attention computation","score":0.6859999895095825},{"id":"https://openalex.org/keywords/matrix-exponential","display_name":"matrix exponential","score":0.3529999852180481},{"id":"https://openalex.org/keywords/subquadratic-algorithms","display_name":"subquadratic algorithms","score":0.6010000109672546},{"id":"https://openalex.org/keywords/additive-approximation","display_name":"additive approximation","score":0.6579999923706055},{"id":"https://openalex.org/keywords/strong-exponential-time-hypothesis","display_name":"Strong Exponential Time Hypothesis","score":0.9340000152587891},{"id":"https://openalex.org/keywords/fine-grained-complexity","display_name":"fine-grained complexity","score":0.9729999899864197},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6119999885559082},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.5789999961853027},{"id":"https://openalex.org/keywords/computational-lower-bounds","display_name":"computational lower bounds","score":0.4650000035762787}],"concepts":[{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.6525999903678894},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.5958999991416931},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.5863000154495239},{"id":"https://openalex.org/C311688","wikidata":"https://www.wikidata.org/wiki/Q2393193","display_name":"Time complexity","level":2,"score":0.5622000098228455},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5590000152587891},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5386000275611877},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49880000948905945},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4415999948978424},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.43389999866485596},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.4275999963283539},{"id":"https://openalex.org/C17349429","wikidata":"https://www.wikidata.org/wiki/Q1049914","display_name":"Matrix multiplication","level":3,"score":0.4214000105857849},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.4106000065803528},{"id":"https://openalex.org/C151376022","wikidata":"https://www.wikidata.org/wiki/Q168698","display_name":"Exponential function","level":2,"score":0.392300009727478},{"id":"https://openalex.org/C195906000","wikidata":"https://www.wikidata.org/wiki/Q1191722","display_name":"Matrix exponential","level":3,"score":0.39010000228881836},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.3855000138282776},{"id":"https://openalex.org/C184596265","wikidata":"https://www.wikidata.org/wiki/Q2651576","display_name":"Model of computation","level":3,"score":0.36980000138282776},{"id":"https://openalex.org/C90119067","wikidata":"https://www.wikidata.org/wiki/Q43260","display_name":"Polynomial","level":2,"score":0.3686999976634979},{"id":"https://openalex.org/C126352355","wikidata":"https://www.wikidata.org/wiki/Q2562273","display_name":"Polynomial matrix","level":4,"score":0.34220001101493835},{"id":"https://openalex.org/C2780427248","wikidata":"https://www.wikidata.org/wiki/Q17014996","display_name":"Fundamental matrix (linear differential equation)","level":2,"score":0.32359999418258667},{"id":"https://openalex.org/C49555168","wikidata":"https://www.wikidata.org/wiki/Q176583","display_name":"Stochastic matrix","level":3,"score":0.29589998722076416},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.28790000081062317},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.28600001335144043},{"id":"https://openalex.org/C3018263672","wikidata":"https://www.wikidata.org/wiki/Q1296251","display_name":"Efficient algorithm","level":2,"score":0.2700999975204468},{"id":"https://openalex.org/C71017364","wikidata":"https://www.wikidata.org/wiki/Q5421532","display_name":"Exponential time hypothesis","level":3,"score":0.267300009727478},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.2651999890804291},{"id":"https://openalex.org/C50335755","wikidata":"https://www.wikidata.org/wiki/Q483247","display_name":"Phenomenon","level":2,"score":0.26269999146461487},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2549000084400177}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52202/075280-2755","doi":"https://doi.org/10.52202/075280-2755","is_oa":false,"landing_page_url":"https://doi.org/10.52202/075280-2755","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":"https://openalex.org/P4404678063","host_organization_name":"Neural Information Processing Systems Foundation","host_organization_lineage":["https://openalex.org/P4404678063"],"host_organization_lineage_names":["Neural Information Processing Systems Foundation"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 36","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2302.13214","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2302.13214","pdf_url":"https://arxiv.org/pdf/2302.13214","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2302.13214","doi":"https://doi.org/10.48550/arxiv.2302.13214","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2302.13214","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2302.13214","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2302.13214","pdf_url":"https://arxiv.org/pdf/2302.13214","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7133196742.pdf","grobid_xml":"https://content.openalex.org/works/W7133196742.grobid-xml"},"referenced_works_count":22,"referenced_works":["https://openalex.org/W1985572324","https://openalex.org/W2769182151","https://openalex.org/W2794237923","https://openalex.org/W2884285311","https://openalex.org/W2888383612","https://openalex.org/W2896457183","https://openalex.org/W2940744433","https://openalex.org/W2970100546","https://openalex.org/W2972926935","https://openalex.org/W2979314664","https://openalex.org/W2994673210","https://openalex.org/W3033529678","https://openalex.org/W3034573343","https://openalex.org/W3092852584","https://openalex.org/W3105990569","https://openalex.org/W3126911039","https://openalex.org/W3189137649","https://openalex.org/W3214693004","https://openalex.org/W4224308101","https://openalex.org/W4229005866","https://openalex.org/W4280565665","https://openalex.org/W4319452930"],"related_works":["https://openalex.org/W4415326843","https://openalex.org/W4394647444","https://openalex.org/W4414818110","https://openalex.org/W7214949558","https://openalex.org/W4417300250","https://openalex.org/W4416035028","https://openalex.org/W4387595597","https://openalex.org/W7220742053","https://openalex.org/W7162329014","https://openalex.org/W4362679267"],"abstract_inverted_index":{"In":[0,102],"modern":[1],"machine":[2],"learning,":[3],"inner":[4],"product":[5],"attention":[6,87,212],"computation":[7,213],"is":[8,31,47,67,73,100,127,148,183,214],"a":[9,128,202],"fundamental":[10],"task":[11],"for":[12,78,205],"training":[13],"large":[14],"language":[15],"models":[16],"such":[17],"as":[18,33],"Transformer,":[19],"GPT-1,":[20],"BERT,":[21],"GPT-2,":[22],"GPT-3":[23],"and":[24,44,71,90,142,167],"ChatGPT.":[25],"Formally,":[26],"in":[27,193,209],"this":[28,79,103],"problem,":[29],"one":[30],"given":[32],"input":[34,220],"three":[35],"matrices":[36,221],"$Q,":[37],"K,":[38],"V":[39,58],"\\in":[40,59],"[-B,B]^{n":[41],"\\times":[42,61,85],"d}$,":[43,62],"the":[45,50,68,83,117,173,206,219],"goal":[46],"to":[48,153,157,185,189],"construct":[49],"matrix":[51,88,118],"$\\mathrm{Att}(Q,K,V)":[52],":=":[53],"\\mathrm{diag}(A":[54],"{\\bf":[55],"1}_n)^{-1}":[56],"A":[57],"\\mathbb{R}^{n":[60],"where":[63],"$A":[64],"=":[65,98,133,139,144,164,169],"\\exp(QK^\\top/d)$":[66],"`attention":[69],"matrix',":[70],"$\\exp$":[72],"applied":[74],"entry-wise.":[75],"Straightforward":[76],"methods":[77],"problem":[80],"explicitly":[81],"compute":[82],"$n":[84],"n$":[86],"$A$,":[89],"hence":[91],"require":[92],"time":[93,151,196],"$\u03a9(n^2)$":[94],"even":[95],"when":[96,218],"$d":[97,138,163],"n^{o(1)}$":[99],"small.":[101],"paper,":[104],"we":[105],"investigate":[106],"whether":[107],"faster":[108],"algorithms":[109],"are":[110],"possible":[111],"by":[112],"implicitly":[113],"making":[114],"use":[115],"of":[116],"$A$.":[119],"We":[120],"present":[121],"two":[122],"results,":[123],"showing":[124],"that":[125,211],"there":[126,147],"sharp":[129],"transition":[130],"at":[131],"$B":[132,143,168],"\u0398(\\sqrt{\\log":[134,170],"n})$.":[135],"$\\bullet$":[136,161],"If":[137,162],"O(\\log":[140,165],"n)$":[141,166],"o(\\sqrt{\\log":[145],"n})$,":[146,171],"an":[149],"$n^{1+o(1)}$":[150],"algorithm":[152],"approximate":[154,186],"$\\mathrm{Att}(Q,K,V)$":[155,187],"up":[156,188],"$1/\\mathrm{poly}(n)$":[158,190],"additive":[159,191],"error.":[160],"assuming":[172],"Strong":[174],"Exponential":[175],"Time":[176],"Hypothesis":[177],"from":[178],"fine-grained":[179],"complexity":[180],"theory,":[181],"it":[182],"impossible":[184],"error":[192],"truly":[194],"subquadratic":[195],"$n^{2":[197],"-":[198],"\u03a9(1)}$.":[199],"This":[200],"gives":[201],"theoretical":[203],"explanation":[204],"phenomenon":[207],"observed":[208],"practice":[210],"much":[215],"more":[216],"efficient":[217],"have":[222],"smaller":[223],"entries.":[224]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7133196742","counts_by_year":[{"year":2026,"cited_by_count":19},{"year":2025,"cited_by_count":42},{"year":2024,"cited_by_count":42},{"year":2023,"cited_by_count":32},{"year":2022,"cited_by_count":3}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2026-03-03T00:00:00"}
