{"id":"https://openalex.org/W4404134155","doi":"https://doi.org/10.1145/3649329.3658253","title":"On the Design of Novel Attention Mechanism for Enhanced Efficiency of Transformers","display_name":"On the Design of Novel Attention Mechanism for Enhanced Efficiency of Transformers","publication_year":2024,"publication_date":"2024-06-23","ids":{"openalex":"https://openalex.org/W4404134155","doi":"https://doi.org/10.1145/3649329.3658253"},"language":"en","primary_location":{"id":"doi:10.1145/3649329.3658253","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3649329.3658253","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 61st ACM/IEEE Design Automation Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075978538","display_name":"Sumit Kumar Jha","orcid":"https://orcid.org/0000-0003-0354-2940"},"institutions":[{"id":"https://openalex.org/I1298353152","display_name":"SRI International","ror":"https://ror.org/05s570m15","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1298353152"]},{"id":"https://openalex.org/I19700959","display_name":"Florida International University","ror":"https://ror.org/02gz6gg07","country_code":"US","type":"education","lineage":["https://openalex.org/I19700959"]},{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sumit Kumar Jha","raw_affiliation_strings":["Computer Science Department, Florida International University, Miami, FL, United States","Computer Science Laboratory, SRI International, Menlo Park, CA, United States"],"raw_orcid":"https://orcid.org/0000-0003-0354-2940","affiliations":[{"raw_affiliation_string":"Computer Science Department, Florida International University, Miami, FL, United States","institution_ids":["https://openalex.org/I19700959"]},{"raw_affiliation_string":"Computer Science Laboratory, SRI International, Menlo Park, CA, United States","institution_ids":["https://openalex.org/I1298353152","https://openalex.org/I4210099336"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035902535","display_name":"Susmit Jha","orcid":"https://orcid.org/0000-0001-5983-9095"},"institutions":[{"id":"https://openalex.org/I1298353152","display_name":"SRI International","ror":"https://ror.org/05s570m15","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1298353152"]},{"id":"https://openalex.org/I19700959","display_name":"Florida International University","ror":"https://ror.org/02gz6gg07","country_code":"US","type":"education","lineage":["https://openalex.org/I19700959"]},{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Susmit Jha","raw_affiliation_strings":["Computer Science Department, Florida International University, Miami, FL, United States","Computer Science Laboratory, SRI International, Menlo Park, CA, United States"],"raw_orcid":"https://orcid.org/0000-0001-5983-9095","affiliations":[{"raw_affiliation_string":"Computer Science Department, Florida International University, Miami, FL, United States","institution_ids":["https://openalex.org/I19700959"]},{"raw_affiliation_string":"Computer Science Laboratory, SRI International, Menlo Park, CA, United States","institution_ids":["https://openalex.org/I1298353152","https://openalex.org/I4210099336"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014892277","display_name":"Rickard Ewetz","orcid":"https://orcid.org/0000-0002-4183-6926"},"institutions":[{"id":"https://openalex.org/I106165777","display_name":"University of Central Florida","ror":"https://ror.org/036nfer12","country_code":"US","type":"education","lineage":["https://openalex.org/I106165777"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rickard Ewetz","raw_affiliation_strings":["Electrical and Computer Engineering, University of Central Florida, Orlando, FL, United States"],"raw_orcid":"https://orcid.org/0000-0002-4183-6926","affiliations":[{"raw_affiliation_string":"Electrical and Computer Engineering, University of Central Florida, Orlando, FL, United States","institution_ids":["https://openalex.org/I106165777"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5067109059","display_name":"Alvaro Velasquez","orcid":"https://orcid.org/0000-0001-6757-105X"},"institutions":[{"id":"https://openalex.org/I188538660","display_name":"University of Colorado Boulder","ror":"https://ror.org/02ttsq026","country_code":"US","type":"education","lineage":["https://openalex.org/I188538660"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alvaro Velasquez","raw_affiliation_strings":["Department of Computer Science, University of Colorado Boulder, Boulder, CO, United States"],"raw_orcid":"https://orcid.org/0000-0001-6757-105X","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Colorado Boulder, Boulder, CO, United States","institution_ids":["https://openalex.org/I188538660"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5075978538"],"corresponding_institution_ids":["https://openalex.org/I1298353152","https://openalex.org/I19700959","https://openalex.org/I4210099336"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.17325191,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12111","display_name":"Industrial Vision Systems and Defect Detection","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12169","display_name":"Non-Destructive Testing Techniques","score":0.9833999872207642,"subfield":{"id":"https://openalex.org/subfields/2210","display_name":"Mechanical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mechanism","display_name":"Mechanism (biology)","score":0.6344862580299377},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5624900460243225},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5125315189361572},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.1925341784954071},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.17472150921821594},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.10640296339988708},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.08205097913742065}],"concepts":[{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.6344862580299377},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5624900460243225},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5125315189361572},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.1925341784954071},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.17472150921821594},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.10640296339988708},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.08205097913742065},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3649329.3658253","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3649329.3658253","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 61st ACM/IEEE Design Automation Conference","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Affordable and clean energy","score":0.8100000023841858,"id":"https://metadata.un.org/sdg/7"}],"awards":[{"id":"https://openalex.org/G5999334576","display_name":null,"funder_award_id":"2408925 2404036","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W258782345","https://openalex.org/W2120000030","https://openalex.org/W2334819541","https://openalex.org/W2795584069","https://openalex.org/W3135547017","https://openalex.org/W3211525823","https://openalex.org/W4206688953","https://openalex.org/W4293023328","https://openalex.org/W4293024063","https://openalex.org/W4293025835","https://openalex.org/W6702889677"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"We":[0,44],"present":[1],"a":[2],"new":[3,148],"xor-based":[4,67,107,134,159],"attention":[5,16,38,68,111,128,135,160],"function":[6,39,112,161],"for":[7,138],"efficient":[8],"hardware":[9,131],"implementation":[10],"of":[11,28,36,50,73,109,141,166],"transformers.":[12,167],"While":[13],"the":[14,23,26,29,34,47,51,66,71,82,110,125,139,158],"standard":[15,52],"mechanism":[17],"relies":[18],"on":[19],"matrix":[20],"multiplication":[21],"between":[22,81],"key":[24],"and":[25,61,92,120],"transpose":[27],"query,":[30],"we":[31,95],"propose":[32],"replacing":[33],"computation":[35,108],"this":[37],"with":[40],"bitwise":[41],"xor":[42],"operations.":[43],"mathematically":[45],"analyze":[46],"information-theoretic":[48],"properties":[49],"multiplication-based":[53,127],"attention,":[54],"demonstrating":[55],"that":[56,65],"it":[57],"preserves":[58,70],"input":[59,75],"entropy,":[60],"then":[62],"computationally":[63],"show":[64,96],"approximately":[69],"entropy":[72],"its":[74],"despite":[76],"small":[77],"variations":[78],"in":[79,116,151],"correlations":[80],"inputs.":[83],"Across":[84],"various":[85],"admittedly":[86],"simple":[87],"tasks,":[88],"including":[89],"arithmetic,":[90],"sorting,":[91],"text":[93],"generation,":[94],"comparable":[97],"performance":[98],"to":[99,124,157],"baseline":[100],"methods":[101],"using":[102],"scaled":[103],"GPT":[104],"models.":[105],"The":[106],"shows":[113],"substantial":[114],"improvement":[115],"power":[117],"consumption,":[118],"latency,":[119],"circuit":[121],"area":[122],"compared":[123],"corresponding":[126],"function.":[129],"This":[130],"efficiency":[132,165],"makes":[133],"more":[136],"compelling":[137],"deployment":[140],"transformers":[142],"under":[143],"tight":[144],"resource":[145],"constraints,":[146],"opening":[147],"application":[149],"domains":[150],"sustainable":[152],"energy-efficient":[153],"computing.":[154],"Additional":[155],"optimizations":[156],"can":[162],"further":[163],"improve":[164]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}