{"id":"https://openalex.org/W4386859272","doi":"https://doi.org/10.1109/islped58423.2023.10244348","title":"ITA: An Energy-Efficient Attention and Softmax Accelerator for Quantized Transformers","display_name":"ITA: An Energy-Efficient Attention and Softmax Accelerator for Quantized Transformers","publication_year":2023,"publication_date":"2023-08-07","ids":{"openalex":"https://openalex.org/W4386859272","doi":"https://doi.org/10.1109/islped58423.2023.10244348"},"language":"en","primary_location":{"id":"doi:10.1109/islped58423.2023.10244348","is_oa":false,"landing_page_url":"https://doi.org/10.1109/islped58423.2023.10244348","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE/ACM International Symposium on Low Power Electronics and Design (ISLPED)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073926165","display_name":"Gamze \u0130slamo\u011flu","orcid":"https://orcid.org/0000-0002-5129-1691"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Gamze Islamoglu","raw_affiliation_strings":["ETH Z&#x00FC;rich,Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Z&#x00FC;rich,Switzerland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017396450","display_name":"Moritz Scherer","orcid":"https://orcid.org/0000-0002-2762-2307"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Moritz Scherer","raw_affiliation_strings":["ETH Z&#x00FC;rich,Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Z&#x00FC;rich,Switzerland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042519536","display_name":"Gianna Paulin","orcid":"https://orcid.org/0000-0002-1310-0911"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gianna Paulin","raw_affiliation_strings":["ETH Z&#x00FC;rich,Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Z&#x00FC;rich,Switzerland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103019315","display_name":"Tim Fischer","orcid":"https://orcid.org/0009-0007-9700-1286"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tim Fischer","raw_affiliation_strings":["ETH Z&#x00FC;rich,Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Z&#x00FC;rich,Switzerland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040009254","display_name":"Victor J. B. Jung","orcid":"https://orcid.org/0009-0001-7462-3468"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Victor J.B. Jung","raw_affiliation_strings":["ETH Z&#x00FC;rich,Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Z&#x00FC;rich,Switzerland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052915995","display_name":"Angelo Garofalo","orcid":"https://orcid.org/0000-0002-7495-6895"},"institutions":[{"id":"https://openalex.org/I9360294","display_name":"University of Bologna","ror":"https://ror.org/01111rn36","country_code":"IT","type":"education","lineage":["https://openalex.org/I9360294"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Angelo Garofalo","raw_affiliation_strings":["ETH Z&#x00FC;rich,Switzerland","University of Bologna, Italy"],"affiliations":[{"raw_affiliation_string":"ETH Z&#x00FC;rich,Switzerland","institution_ids":[]},{"raw_affiliation_string":"University of Bologna, Italy","institution_ids":["https://openalex.org/I9360294"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043408422","display_name":"Luca Benini","orcid":"https://orcid.org/0000-0001-8068-3806"},"institutions":[{"id":"https://openalex.org/I9360294","display_name":"University of Bologna","ror":"https://ror.org/01111rn36","country_code":"IT","type":"education","lineage":["https://openalex.org/I9360294"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Luca Benini","raw_affiliation_strings":["ETH Z&#x00FC;rich,Switzerland","University of Bologna, Italy"],"affiliations":[{"raw_affiliation_string":"ETH Z&#x00FC;rich,Switzerland","institution_ids":[]},{"raw_affiliation_string":"University of Bologna, Italy","institution_ids":["https://openalex.org/I9360294"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5073926165"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.7342,"has_fulltext":false,"cited_by_count":19,"citation_normalized_percentile":{"value":0.96860713,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6528695225715637},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6454161405563354},{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.6405820250511169},{"id":"https://openalex.org/keywords/efficient-energy-use","display_name":"Efficient energy use","score":0.5803271532058716},{"id":"https://openalex.org/keywords/tops","display_name":"TOPS","score":0.506866991519928},{"id":"https://openalex.org/keywords/dataflow","display_name":"Dataflow","score":0.4694214463233948},{"id":"https://openalex.org/keywords/interfacing","display_name":"Interfacing","score":0.43678930401802063},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.3897726833820343},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.3813370168209076},{"id":"https://openalex.org/keywords/computer-hardware","display_name":"Computer hardware","score":0.33434784412384033},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.2515823245048523},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.2374674677848816},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.229048490524292},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.15672022104263306},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1470898985862732}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6528695225715637},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6454161405563354},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.6405820250511169},{"id":"https://openalex.org/C2742236","wikidata":"https://www.wikidata.org/wiki/Q924713","display_name":"Efficient energy use","level":2,"score":0.5803271532058716},{"id":"https://openalex.org/C2777675136","wikidata":"https://www.wikidata.org/wiki/Q835642","display_name":"TOPS","level":3,"score":0.506866991519928},{"id":"https://openalex.org/C96324660","wikidata":"https://www.wikidata.org/wiki/Q205446","display_name":"Dataflow","level":2,"score":0.4694214463233948},{"id":"https://openalex.org/C2776303644","wikidata":"https://www.wikidata.org/wiki/Q1020499","display_name":"Interfacing","level":2,"score":0.43678930401802063},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.3897726833820343},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.3813370168209076},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.33434784412384033},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.2515823245048523},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.2374674677848816},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.229048490524292},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.15672022104263306},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1470898985862732},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C154815118","wikidata":"https://www.wikidata.org/wiki/Q453762","display_name":"Spinning","level":2,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/islped58423.2023.10244348","is_oa":false,"landing_page_url":"https://doi.org/10.1109/islped58423.2023.10244348","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE/ACM International Symposium on Low Power Electronics and Design (ISLPED)","raw_type":"proceedings-article"},{"id":"pmh:oai:cris.unibo.it:11585/958810","is_oa":false,"landing_page_url":"https://hdl.handle.net/11585/958810","pdf_url":null,"source":{"id":"https://openalex.org/S4306402579","display_name":"Archivio istituzionale della ricerca (Alma Mater Studiorum Universit\u00e0 di Bologna)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210117483","host_organization_name":"Istituto di Ematologia di Bologna","host_organization_lineage":["https://openalex.org/I4210117483"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/7","display_name":"Affordable and clean energy","score":0.9100000262260437}],"awards":[{"id":"https://openalex.org/G5022591845","display_name":null,"funder_award_id":"101070634","funder_id":"https://openalex.org/F4320334322","funder_display_name":"HORIZON EUROPE Framework Programme"}],"funders":[{"id":"https://openalex.org/F4320334322","display_name":"HORIZON EUROPE Framework Programme","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W2896457183","https://openalex.org/W3037132819","https://openalex.org/W3094502228","https://openalex.org/W3126721948","https://openalex.org/W3158129762","https://openalex.org/W3159727696","https://openalex.org/W3159778524","https://openalex.org/W3170233084","https://openalex.org/W3177265267","https://openalex.org/W3184107541","https://openalex.org/W3189877953","https://openalex.org/W3197527602","https://openalex.org/W3211525823","https://openalex.org/W4220865834","https://openalex.org/W4286571858","https://openalex.org/W4288337707","https://openalex.org/W4385245566","https://openalex.org/W6780125659","https://openalex.org/W6809871064"],"related_works":["https://openalex.org/W2811109569","https://openalex.org/W2385933648","https://openalex.org/W2381557761","https://openalex.org/W3107204728","https://openalex.org/W4287591324","https://openalex.org/W4226420367","https://openalex.org/W2980176872","https://openalex.org/W2962876041","https://openalex.org/W2998381397","https://openalex.org/W3090555870"],"abstract_inverted_index":{"Transformer":[0],"networks":[1],"have":[2],"emerged":[3],"as":[4,21],"the":[5,28],"state-of-the-art":[6,111],"approach":[7],"for":[8,61],"natural":[9],"language":[10],"processing":[11],"tasks":[12],"and":[13,24,47,63,77,100],"are":[14],"gaining":[15],"popularity":[16],"in":[17,91,120,129],"other":[18],"domains":[19],"such":[20],"computer":[22],"vision":[23],"audio":[25],"processing.":[26],"However,":[27],"efficient":[29,68],"hardware":[30],"acceleration":[31],"of":[32],"transformer":[33,112],"models":[34,65],"poses":[35],"new":[36],"challenges":[37],"due":[38],"to":[39,110],"their":[40],"high":[41],"arithmetic":[42],"intensities,":[43],"large":[44],"memory":[45],"requirements,":[46],"complex":[48],"dataflow":[49],"dependencies.":[50],"In":[51],"this":[52],"work,":[53],"we":[54],"propose":[55],"ITA,":[56],"a":[57],"novel":[58],"accelerator":[59],"architecture":[60],"transformers":[62],"related":[64],"that":[66,82],"targets":[67],"inference":[69],"on":[70,85],"embedded":[71],"systems":[72],"by":[73],"exploiting":[74],"8-bit":[75],"quantization":[76],"an":[78],"innovative":[79],"softmax":[80,95],"implementation":[81,96],"operates":[83],"exclusively":[84],"integer":[86],"values.":[87],"By":[88],"computing":[89],"on-the-fly":[90],"streaming":[92],"mode,":[93],"our":[94],"minimizes":[97],"data":[98],"movement":[99],"energy":[101,106],"consumption.":[102],"ITA":[103],"achieves":[104],"competitive":[105],"efficiency":[107,122],"with":[108,114,123],"respect":[109],"accelerators":[113],"16.9":[115],"TOPS/W,":[116],"while":[117],"outperforming":[118],"them":[119],"area":[121],"5.93":[124],"TOPS/mm":[125],"<sup":[126],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[127],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">2</sup>":[128],"22":[130],"nm":[131],"fully-depleted":[132],"silicon-on-insulator":[133],"technology":[134],"at":[135],"0.8":[136],"V.":[137]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":6}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2023-09-20T00:00:00"}