{"id":"https://openalex.org/W4402451456","doi":"https://doi.org/10.1109/tcasai.2024.3459412","title":"Optimizing Foundation Model Inference on a Many-Tiny-Core Open-Source RISC-V Platform","display_name":"Optimizing Foundation Model Inference on a Many-Tiny-Core Open-Source RISC-V Platform","publication_year":2024,"publication_date":"2024-09-01","ids":{"openalex":"https://openalex.org/W4402451456","doi":"https://doi.org/10.1109/tcasai.2024.3459412"},"language":"en","primary_location":{"id":"doi:10.1109/tcasai.2024.3459412","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcasai.2024.3459412","pdf_url":null,"source":{"id":"https://openalex.org/S4404675360","display_name":"IEEE transactions on circuits and systems for artificial intelligence.","issn_l":"2996-6647","issn":["2996-6647"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Artificial Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.1109/TCASAI.2024.3459412","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5062172879","display_name":"Viviane Potocnik","orcid":"https://orcid.org/0009-0004-9412-6081"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Viviane Potocnik","raw_affiliation_strings":["Integrated Systems Laboratory (IIS) of ETH Zürich, ETZ, Züurich, Switzerland"],"raw_orcid":"https://orcid.org/0009-0004-9412-6081","affiliations":[{"raw_affiliation_string":"Integrated Systems Laboratory (IIS) of ETH Zürich, ETZ, Züurich, Switzerland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094583316","display_name":"Luca Colagrande","orcid":"https://orcid.org/0000-0002-7986-1975"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luca Colagrande","raw_affiliation_strings":["Integrated Systems Laboratory (IIS) of ETH Zürich, ETZ, Züurich, Switzerland"],"raw_orcid":"https://orcid.org/0000-0002-7986-1975","affiliations":[{"raw_affiliation_string":"Integrated Systems Laboratory (IIS) of ETH Zürich, ETZ, Züurich, Switzerland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103019315","display_name":"Tim Fischer","orcid":"https://orcid.org/0009-0007-9700-1286"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tim Fischer","raw_affiliation_strings":["Integrated Systems Laboratory (IIS) of ETH Zürich, ETZ, Züurich, Switzerland"],"raw_orcid":"https://orcid.org/0009-0007-9700-1286","affiliations":[{"raw_affiliation_string":"Integrated Systems Laboratory (IIS) of ETH Zürich, ETZ, Züurich, Switzerland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062088050","display_name":"Luca Bertaccini","orcid":"https://orcid.org/0000-0002-3011-6368"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luca Bertaccini","raw_affiliation_strings":["Integrated Systems Laboratory (IIS) of ETH Zürich, ETZ, Züurich, Switzerland"],"raw_orcid":"https://orcid.org/0000-0002-3011-6368","affiliations":[{"raw_affiliation_string":"Integrated Systems Laboratory (IIS) of ETH Zürich, ETZ, Züurich, Switzerland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080843266","display_name":"Daniele Jahier Pagliari","orcid":"https://orcid.org/0000-0002-2872-7071"},"institutions":[{"id":"https://openalex.org/I177477856","display_name":"Politecnico di Torino","ror":"https://ror.org/00bgk9508","country_code":"IT","type":"education","lineage":["https://openalex.org/I177477856"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Daniele Jahier Pagliari","raw_affiliation_strings":["Department of Control and Computer Engineering, Politecnico di Torino, Turin, Italy"],"raw_orcid":"https://orcid.org/0000-0002-2872-7071","affiliations":[{"raw_affiliation_string":"Department of Control and Computer Engineering, Politecnico di Torino, Turin, Italy","institution_ids":["https://openalex.org/I177477856"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032095821","display_name":"Alessio Burrello","orcid":"https://orcid.org/0000-0002-6215-8220"},"institutions":[{"id":"https://openalex.org/I9360294","display_name":"University of Bologna","ror":"https://ror.org/01111rn36","country_code":"IT","type":"education","lineage":["https://openalex.org/I9360294"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Alessio Burrello","raw_affiliation_strings":["Department of Electrical, Electronic and Information Engineering, University of Bologna, Bologna, Italy"],"raw_orcid":"https://orcid.org/0000-0002-6215-8220","affiliations":[{"raw_affiliation_string":"Department of Electrical, Electronic and Information Engineering, University of Bologna, Bologna, Italy","institution_ids":["https://openalex.org/I9360294"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043408422","display_name":"Luca Benini","orcid":"https://orcid.org/0000-0001-8068-3806"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luca Benini","raw_affiliation_strings":["Integrated Systems Laboratory (IIS) of ETH Zürich, ETZ, Züurich, Switzerland"],"raw_orcid":"https://orcid.org/0000-0001-8068-3806","affiliations":[{"raw_affiliation_string":"Integrated Systems Laboratory (IIS) of ETH Zürich, ETZ, Züurich, Switzerland","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.8733,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.86732409,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"1","issue":"1","first_page":"37","last_page":"52"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9368000030517578,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9368000030517578,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/open-source","display_name":"Open source","score":0.7188918590545654},{"id":"https://openalex.org/keywords/foundation","display_name":"Foundation (evidence)","score":0.6579675674438477},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6096251606941223},{"id":"https://openalex.org/keywords/core","display_name":"Core (optical fiber)","score":0.5867245197296143},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5000603199005127},{"id":"https://openalex.org/keywords/open-platform","display_name":"Open platform","score":0.4911174476146698},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.46064406633377075},{"id":"https://openalex.org/keywords/core-model","display_name":"Core model","score":0.41185128688812256},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3640097975730896},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.21395015716552734},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.12525251507759094},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.11000418663024902},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10940206050872803},{"id":"https://openalex.org/keywords/history","display_name":"History","score":0.09964671730995178},{"id":"https://openalex.org/keywords/archaeology","display_name":"Archaeology","score":0.07763656973838806}],"concepts":[{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.7188918590545654},{"id":"https://openalex.org/C2780966255","wikidata":"https://www.wikidata.org/wiki/Q5474306","display_name":"Foundation (evidence)","level":2,"score":0.6579675674438477},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6096251606941223},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.5867245197296143},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5000603199005127},{"id":"https://openalex.org/C108383078","wikidata":"https://www.wikidata.org/wiki/Q7096399","display_name":"Open platform","level":3,"score":0.4911174476146698},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.46064406633377075},{"id":"https://openalex.org/C2780366209","wikidata":"https://www.wikidata.org/wiki/Q5170200","display_name":"Core model","level":2,"score":0.41185128688812256},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3640097975730896},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21395015716552734},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.12525251507759094},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.11000418663024902},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10940206050872803},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.09964671730995178},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.07763656973838806},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tcasai.2024.3459412","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcasai.2024.3459412","pdf_url":null,"source":{"id":"https://openalex.org/S4404675360","display_name":"IEEE transactions on circuits and systems for artificial intelligence.","issn_l":"2996-6647","issn":["2996-6647"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:cris.unibo.it:11585/1004729","is_oa":false,"landing_page_url":"https://hdl.handle.net/11585/1004729","pdf_url":null,"source":{"id":"https://openalex.org/S4306402579","display_name":"Archivio istituzionale della ricerca (Alma Mater Studiorum Universit\u00e0 di Bologna)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210117483","host_organization_name":"Istituto di Ematologia di Bologna","host_organization_lineage":["https://openalex.org/I4210117483"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:zenodo.org:15624217","is_oa":true,"landing_page_url":"https://doi.org/10.1109/TCASAI.2024.3459412","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE TRANSACTIONS ON CIRCUITS AND SYSTEMS FOR ARTIFICIAL INTELLIGENCE, 1(1), 37-52, (2024-09-11)","raw_type":"info:eu-repo/semantics/conferencePaper"}],"best_oa_location":{"id":"pmh:oai:zenodo.org:15624217","is_oa":true,"landing_page_url":"https://doi.org/10.1109/TCASAI.2024.3459412","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE TRANSACTIONS ON CIRCUITS AND SYSTEMS FOR ARTIFICIAL INTELLIGENCE, 1(1), 37-52, (2024-09-11)","raw_type":"info:eu-repo/semantics/conferencePaper"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":47,"referenced_works":["https://openalex.org/W2896457183","https://openalex.org/W2963250244","https://openalex.org/W2981980078","https://openalex.org/W3013692244","https://openalex.org/W3112773671","https://openalex.org/W3130554079","https://openalex.org/W3204538018","https://openalex.org/W4214493665","https://openalex.org/W4281708879","https://openalex.org/W4297097382","https://openalex.org/W4312639064","https://openalex.org/W4313467238","https://openalex.org/W4324292875","https://openalex.org/W4362707004","https://openalex.org/W4367595583","https://openalex.org/W4376123125","https://openalex.org/W4376130831","https://openalex.org/W4385245566","https://openalex.org/W4388469806","https://openalex.org/W4391944268","https://openalex.org/W4394597798","https://openalex.org/W4396909834","https://openalex.org/W4401880104","https://openalex.org/W6753770798","https://openalex.org/W6763653508","https://openalex.org/W6778883912","https://openalex.org/W6779163297","https://openalex.org/W6779709467","https://openalex.org/W6784333009","https://openalex.org/W6788001715","https://openalex.org/W6796753453","https://openalex.org/W6800751262","https://openalex.org/W6839178539","https://openalex.org/W6843734122","https://openalex.org/W6846659131","https://openalex.org/W6850462617","https://openalex.org/W6851027815","https://openalex.org/W6851059687","https://openalex.org/W6851775633","https://openalex.org/W6852408377","https://openalex.org/W6853251322","https://openalex.org/W6854308872","https://openalex.org/W6854866820","https://openalex.org/W6857524558","https://openalex.org/W6859495724","https://openalex.org/W6860380251","https://openalex.org/W6861768399"],"related_works":["https://openalex.org/W177188689","https://openalex.org/W2022349550","https://openalex.org/W2377213713","https://openalex.org/W4255874227","https://openalex.org/W3004249933","https://openalex.org/W4242732953","https://openalex.org/W4402405851","https://openalex.org/W3022089151","https://openalex.org/W2795267055","https://openalex.org/W2363532283"],"abstract_inverted_index":{"Transformer-based":[0],"foundation":[1],"models":[2,20,58],"have":[3],"become":[4],"crucial":[5],"for":[6,73],"various":[7],"domains,":[8],"most":[9,125],"notably":[10],"natural":[11],"language":[12],"processing":[13],"(NLP)":[14],"or":[15,27],"computer":[16],"vision":[17],"(CV).":[18],"These":[19],"are":[21],"predominantly":[22],"deployed":[23],"on":[24,59,100],"high-performance":[25],"GPUs":[26],"hardwired":[28],"accelerators":[29,144],"with":[30],"highly":[31],"customized,":[32],"proprietary":[33],"instruction":[34,79],"sets.":[35],"Until":[36],"now,":[37],"limited":[38],"attention":[39],"has":[40],"been":[41],"given":[42],"to":[43,87,94,118,180,192,197],"RISC-V-based":[44],"general-purpose":[45],"platforms.":[46],"In":[47],"our":[48],"work,":[49],"we":[50,112,166,203],"present":[51],"the":[52,124,129,153,174,187,193,198],"first":[53],"inference":[54],"results":[55],"of":[56,116],"transformer":[57,103],"an":[60],"open-source":[61],"many-tiny-core":[62],"RISC-V":[63],"platform":[64,155],"implementing":[65],"distributed":[66],"Softmax":[67],"primitives":[68],"and":[69,78,93,106,128,138,178],"leveraging":[70],"ISA":[71],"extensions":[72],"SIMD":[74],"floating-point":[75],"operand":[76],"streaming":[77],"repetition,":[80],"as":[81,83],"well":[82],"specialized":[84],"DMA":[85],"engines":[86],"minimize":[88],"costly":[89],"main":[90],"memory":[91],"accesses":[92],"tolerate":[95],"their":[96],"latency.":[97],"We":[98,132],"focus":[99],"two":[101],"foundational":[102],"topologies,":[104,165],"encoder-only":[105,110],"decoder-only":[107,164],"models.":[108],"For":[109,163],"models,":[111],"demonstrate":[113],"a":[114],"speedup":[115,172,185],"up":[117,179],"12.8$\\times$":[122,151,171,184,208],"between":[123],"optimized":[126],"implementation":[127],"baseline":[130,194],"version.":[131],"reach":[133],"over":[134],"79%":[135],"FPU":[136,210],"utilization":[137],"294":[139],"GFLOPS/W,":[140],"outperforming":[141],"State-of-the-Art":[142],"(SoA)":[143],"by":[145],"more":[146],"than":[147],"2