{"id":"https://openalex.org/W7156680405","doi":"https://doi.org/10.48550/arxiv.2604.23798","title":"ELSA: Exact Linear-Scan Attention for Fast and Memory-Light Vision Transformers","display_name":"ELSA: Exact Linear-Scan Attention for Fast and Memory-Light Vision Transformers","publication_year":2026,"publication_date":"2026-04-26","ids":{"openalex":"https://openalex.org/W7156680405","doi":"https://doi.org/10.48550/arxiv.2604.23798"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.23798","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.23798","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.23798","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007305393","display_name":"Chih\u2013Chung Hsu","orcid":"https://orcid.org/0000-0002-2083-4438"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hsu, Chih-Chung","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080208218","display_name":"Xindi Ma","orcid":"https://orcid.org/0000-0001-7129-3814"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Xin-Di","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045264381","display_name":"Wo-Ting Liao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liao, Wo-Ting","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5034075233","display_name":"Chia-Ming Lee","orcid":"https://orcid.org/0009-0004-6027-3083"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Chia-Ming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.38019999861717224,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.38019999861717224,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.08569999784231186,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11992","display_name":"CCD and CMOS Imaging Sensors","score":0.054499998688697815,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.9000999927520752},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.44190001487731934},{"id":"https://openalex.org/keywords/error-detection-and-correction","display_name":"Error detection and correction","score":0.37279999256134033},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.37279999256134033},{"id":"https://openalex.org/keywords/operand","display_name":"Operand","score":0.36489999294281006},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.3433000147342682},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.3384999930858612},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.32820001244544983}],"concepts":[{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.9000999927520752},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6601999998092651},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.44190001487731934},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.41830000281333923},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.37279999256134033},{"id":"https://openalex.org/C103088060","wikidata":"https://www.wikidata.org/wiki/Q1062839","display_name":"Error detection and correction","level":2,"score":0.37279999256134033},{"id":"https://openalex.org/C55526617","wikidata":"https://www.wikidata.org/wiki/Q719375","display_name":"Operand","level":2,"score":0.36489999294281006},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.35269999504089355},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.34369999170303345},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.3433000147342682},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.3384999930858612},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.3296999931335449},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.32820001244544983},{"id":"https://openalex.org/C78023250","wikidata":"https://www.wikidata.org/wiki/Q657596","display_name":"Unary operation","level":2,"score":0.3255999982357025},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.30959999561309814},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.29739999771118164},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.28130000829696655},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.2784000039100647},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27570000290870667},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.27489998936653137},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.27390000224113464},{"id":"https://openalex.org/C76752949","wikidata":"https://www.wikidata.org/wiki/Q7607499","display_name":"Stencil","level":2,"score":0.26930001378059387},{"id":"https://openalex.org/C42935608","wikidata":"https://www.wikidata.org/wiki/Q190411","display_name":"Field-programmable gate array","level":2,"score":0.26350000500679016},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.26249998807907104},{"id":"https://openalex.org/C159423971","wikidata":"https://www.wikidata.org/wiki/Q177251","display_name":"Associative property","level":2,"score":0.25529998540878296},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.25209999084472656},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.25130000710487366}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.23798","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.23798","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.23798","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.23798","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Existing":[0],"attention":[1,34],"accelerators":[2],"often":[3],"trade":[4],"exact":[5,37],"softmax":[6,33,38,55],"semantics,":[7],"depend":[8],"on":[9,22,101,115,158,160],"fused":[10],"Tensor":[11,103],"Core":[12,104],"kernels,":[13],"or":[14,94],"incur":[15],"sequential":[16],"depth":[17,136],"that":[18,35,133],"limits":[19],"FP32":[20,48,110,145,191],"throughput":[21,172],"long":[23,186],"sequences.":[24],"We":[25],"present":[26],"\\textbf{ELSA},":[27],"an":[28,62],"algorithmic":[29],"reformulation":[30],"of":[31],"online":[32,54],"(i)~preserves":[36],"semantics":[39],"in":[40,80],"real":[41],"arithmetic":[42],"with":[43,170],"a":[44,58,88,194],"\\emph{provable}":[45],"$\\mathcal{O}(u\\log":[46],"n)$":[47,72,139],"relative":[49],"error":[50],"bound;":[51],"(ii)~casts":[52],"the":[53,128],"update":[56],"as":[57,87,122],"prefix":[59],"scan":[60],"over":[61,153,166],"associative":[63],"monoid":[64],"$(m,S,W)$,":[65],"yielding":[66],"$O(n)$":[67],"extra":[68],"memory":[69],"and":[70,75,82,85,106,117,156,204],"$O(\\log":[71,138],"parallel":[73,135],"depth;":[74],"(iii)~is":[76],"Tensor-Core":[77],"independent,":[78],"implemented":[79],"Triton":[81],"CUDA":[83],"C++,":[84],"deployable":[86],"\\emph{drop-in":[89],"replacement}":[90],"requiring":[91],"no":[92,108],"retraining":[93],"weight":[95],"modification.":[96],"Unlike":[97],"FlashAttention-2/3,":[98],"which":[99],"rely":[100],"HMMA/GMMA":[102],"instructions":[105],"provide":[107],"compatible":[109],"path,":[111],"ELSA":[112,149,163,181],"operates":[113],"identically":[114],"A100s":[116],"resource-constrained":[118],"edge":[119],"devices":[120],"such":[121],"Jetson":[123,161],"TX2":[124],"--":[125],"making":[126],"it":[127],"only":[129],"hardware-agnostic":[130],"exact-attention":[131],"kernel":[132,196],"reduces":[134],"to":[137],"at":[140,177,185,208],"full":[141,190],"precision.":[142],"On":[143],"A100":[144],"benchmarks":[146],"(1K--16K":[147],"tokens),":[148,169],"delivers":[150],"$1.3$--$3.5\\times$":[151],"speedup":[152],"memory-efficient":[154],"SDPA":[155],"$1.97$--$2.27\\times$":[157],"BERT;":[159],"TX2,":[162],"achieves":[164],"$1.5$--$1.6\\times$":[165],"Math":[167],"(64--900":[168],"$17.8$--$20.2\\%$":[171],"gains":[173],"under":[174],"LLaMA-13B":[175],"offloading":[176],"$\\ge$32K.":[178],"In":[179],"FP16,":[180],"approaches":[182],"hardware-fused":[183],"baselines":[184],"sequences":[187],"while":[188],"retaining":[189],"capability,":[192],"offering":[193],"unified":[195],"for":[197],"high-precision":[198],"inference":[199],"across":[200],"platforms.":[201],"Our":[202],"code":[203],"implementation":[205],"are":[206],"available":[207],"https://github.com/ming053l/ELSA.":[209]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-29T00:00:00"}