{"id":"https://openalex.org/W4415797154","doi":"https://doi.org/10.52202/079017-2193","title":"FlashAttention-3: Fast and Accurate Attention with Asynchrony and Low-precision","display_name":"FlashAttention-3: Fast and Accurate Attention with Asynchrony and Low-precision","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4415797154","doi":"https://doi.org/10.52202/079017-2193"},"language":"en","primary_location":{"id":"doi:10.52202/079017-2193","doi":"https://doi.org/10.52202/079017-2193","is_oa":false,"landing_page_url":"https://doi.org/10.52202/079017-2193","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 37","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.08608","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Jay Shah","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jay Shah","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086155148","display_name":"Ganesh Bikshandi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ganesh Bikshandi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ying Zhang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ying Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086615721","display_name":"Vijay Thakkar","orcid":"https://orcid.org/0000-0002-6158-6127","observed_orcids":["https://orcid.org/0000-0002-6158-6127"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vijay Thakkar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104453412","display_name":"Pradeep Ramani","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pradeep Ramani","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5091734792","display_name":"Tri Dao","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tri Dao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":31.5024,"has_fulltext":true,"cited_by_count":104,"citation_normalized_percentile":{"value":0.99826389,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"68658","last_page":"68685"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.4740999937057495,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.4740999937057495,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.27079999446868896,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.02500000037252903,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/attention-mechanism","display_name":"attention mechanism","score":0.6330000162124634},{"id":"https://openalex.org/keywords/asynchrony","display_name":"asynchrony","score":0.7760000228881836},{"id":"https://openalex.org/keywords/low-precision-arithmetic","display_name":"low-precision arithmetic","score":0.9070000052452087},{"id":"https://openalex.org/keywords/inference-efficiency","display_name":"inference efficiency","score":0.38600000739097595}],"concepts":[{"id":"https://openalex.org/C2779019669","wikidata":"https://www.wikidata.org/wiki/Q25203946","display_name":"Asynchrony (computer programming)","level":3,"score":0.6789000034332275},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5440000295639038},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35910001397132874},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.2705000042915344},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.23149999976158142},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.22930000722408295},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.22910000383853912},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.22010000050067902},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.21660000085830688},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2126999944448471}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52202/079017-2193","doi":"https://doi.org/10.52202/079017-2193","is_oa":false,"landing_page_url":"https://doi.org/10.52202/079017-2193","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 37","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2407.08608","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.08608","pdf_url":"https://arxiv.org/pdf/2407.08608","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.08608","doi":"https://doi.org/10.48550/arxiv.2407.08608","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.08608","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.08608","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.08608","pdf_url":"https://arxiv.org/pdf/2407.08608","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415797154.pdf","grobid_xml":"https://content.openalex.org/works/W4415797154.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7134042529","https://openalex.org/W4384648639","https://openalex.org/W4404261157","https://openalex.org/W4390042124","https://openalex.org/W7215098400","https://openalex.org/W7125919258","https://openalex.org/W4415968375","https://openalex.org/W7203934078","https://openalex.org/W7214711324","https://openalex.org/W4417299811"],"abstract_inverted_index":{"Attention,":[0],"as":[1],"a":[2,151],"core":[3],"layer":[4],"of":[5,41,72],"the":[6,11,55,73],"ubiquitous":[7],"Transformer":[8],"architecture,":[9],"is":[10],"bottleneck":[12],"for":[13,107],"large":[14],"language":[15],"models":[16],"and":[17,76,83,88,93,96,100,132],"long-context":[18],"applications.":[19],"FlashAttention":[20],"elaborated":[21],"an":[22],"approach":[23],"to":[24,38,63,78,127,137],"speed":[25,64],"up":[26,65,126],"attention":[27,66],"on":[28,54,67,118],"GPUs":[29,120],"through":[30],"minimizing":[31],"memory":[32],"reads/writes.":[33],"However,":[34],"it":[35],"has":[36],"yet":[37],"take":[39],"advantage":[40],"new":[42],"capabilities":[43],"present":[44],"in":[45],"recent":[46],"hardware,":[47],"with":[48,123,133],"FlashAttention-2":[49],"achieving":[50],"only":[51],"35%":[52],"utilization":[53],"H100":[56,119],"GPU.":[57],"We":[58,110,140],"develop":[59],"three":[60],"main":[61],"techniques":[62],"Hopper":[68],"GPUs:":[69],"exploiting":[70],"asynchrony":[71],"Tensor":[74],"Cores":[75],"TMA":[77],"(1)":[79],"overlap":[80],"overall":[81],"computation":[82],"data":[84],"movement":[85],"via":[86],"warp-specialization":[87],"(2)":[89],"interleave":[90],"block-wise":[91],"matmul":[92],"softmax":[94],"operations,":[95],"(3)":[97],"block":[98],"quantization":[99],"incoherent":[101],"processing":[102],"that":[103,112,142],"leverages":[104],"hardware":[105],"support":[106],"FP8":[108,134,143,153],"low-precision.":[109],"demonstrate":[111],"our":[113],"method,":[114],"FlashAttention-3,":[115],"achieves":[116,145],"speedup":[117],"by":[121],"1.5-2.0$\\times$":[122],"FP16":[124],"reaching":[125,135],"740":[128],"TFLOPs/s":[129],"(75%":[130],"utilization),":[131],"close":[136],"1.2":[138],"PFLOPs/s.":[139],"validate":[141],"FlashAttention-3":[144],"2.6$\\times$":[146],"lower":[147],"numerical":[148],"error":[149],"than":[150],"baseline":[152],"attention.":[154]},"counts_by_year":[{"year":2026,"cited_by_count":73},{"year":2025,"cited_by_count":29},{"year":2024,"cited_by_count":2}],"updated_date":"2026-11-01T11:40:01.658920","created_date":"2025-11-03T00:00:00"}