{"id":"https://openalex.org/W4409282598","doi":"https://doi.org/10.1145/3676536.3676743","title":"FAS-Trans: Fully Exploiting FFN and Attention Sparsity for Transformer on FPGA","display_name":"FAS-Trans: Fully Exploiting FFN and Attention Sparsity for Transformer on FPGA","publication_year":2024,"publication_date":"2024-10-27","ids":{"openalex":"https://openalex.org/W4409282598","doi":"https://doi.org/10.1145/3676536.3676743"},"language":"en","primary_location":{"id":"doi:10.1145/3676536.3676743","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3676536.3676743","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3676536.3676743","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd IEEE/ACM International Conference on Computer-Aided Design","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3676536.3676743","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5115604891","display_name":"Hongji Wang","orcid":"https://orcid.org/0009-0006-8758-7239"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]},{"id":"https://openalex.org/I4210132426","display_name":"Shanghai Fudan Microelectronics (China)","ror":"https://ror.org/02vfj3j86","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210132426"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongji Wang","raw_affiliation_strings":["School of Microelectronics, Fudan University, Shanghai, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0006-8758-7239","affiliations":[{"raw_affiliation_string":"School of Microelectronics, Fudan University, Shanghai, Shanghai, China","institution_ids":["https://openalex.org/I24943067","https://openalex.org/I4210132426"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015932760","display_name":"Kun Wang","orcid":"https://orcid.org/0000-0002-7288-1789"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]},{"id":"https://openalex.org/I4210132426","display_name":"Shanghai Fudan Microelectronics (China)","ror":"https://ror.org/02vfj3j86","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210132426"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kun Wang","raw_affiliation_strings":["School of Microelectronics, Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-7288-1789","affiliations":[{"raw_affiliation_string":"School of Microelectronics, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067","https://openalex.org/I4210132426"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yifan Zhang","orcid":"https://orcid.org/0009-0007-7571-1914"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]},{"id":"https://openalex.org/I4210132426","display_name":"Shanghai Fudan Microelectronics (China)","ror":"https://ror.org/02vfj3j86","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210132426"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yifan Zhang","raw_affiliation_strings":["School of Microelectronics, Fudan University, Shanghai, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0007-7571-1914","affiliations":[{"raw_affiliation_string":"School of Microelectronics, Fudan University, Shanghai, Shanghai, China","institution_ids":["https://openalex.org/I24943067","https://openalex.org/I4210132426"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103098516","display_name":"Jun Yu","orcid":"https://orcid.org/0000-0003-4286-9292"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]},{"id":"https://openalex.org/I4210132426","display_name":"Shanghai Fudan Microelectronics (China)","ror":"https://ror.org/02vfj3j86","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210132426"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Yu","raw_affiliation_strings":["School of Microelectronics, Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-4286-9292","affiliations":[{"raw_affiliation_string":"School of Microelectronics, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067","https://openalex.org/I4210132426"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.5264,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.7382301,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10688","display_name":"Image and Signal Denoising Methods","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10688","display_name":"Image and Signal Denoising Methods","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12357","display_name":"Digital Media Forensic Detection","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10558","display_name":"Advancements in Semiconductor Devices and Circuit Design","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/field-programmable-gate-array","display_name":"Field-programmable gate array","score":0.7779507637023926},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.549841046333313},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.511896014213562},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.2997491955757141},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.22826096415519714},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.14095094799995422}],"concepts":[{"id":"https://openalex.org/C42935608","wikidata":"https://www.wikidata.org/wiki/Q190411","display_name":"Field-programmable gate array","level":2,"score":0.7779507637023926},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.549841046333313},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.511896014213562},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.2997491955757141},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.22826096415519714},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.14095094799995422},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3676536.3676743","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3676536.3676743","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3676536.3676743","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd IEEE/ACM International Conference on Computer-Aided Design","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3676536.3676743","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3676536.3676743","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3676536.3676743","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd IEEE/ACM International Conference on Computer-Aided Design","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6507269736","display_name":null,"funder_award_id":"2021YFA1003602","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320321851","display_name":"Fudan University","ror":"https://ror.org/013q1eq08"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4409282598.pdf","grobid_xml":"https://content.openalex.org/works/W4409282598.grobid-xml"},"referenced_works_count":22,"referenced_works":["https://openalex.org/W2108598243","https://openalex.org/W2762374354","https://openalex.org/W2896457183","https://openalex.org/W2949320316","https://openalex.org/W2979826702","https://openalex.org/W3017024317","https://openalex.org/W3047848469","https://openalex.org/W3130240120","https://openalex.org/W3155487259","https://openalex.org/W3159727696","https://openalex.org/W3189877953","https://openalex.org/W3205192296","https://openalex.org/W3206837665","https://openalex.org/W4214686755","https://openalex.org/W4293023328","https://openalex.org/W4293025165","https://openalex.org/W4308479898","https://openalex.org/W4313467238","https://openalex.org/W4380881077","https://openalex.org/W4386108348","https://openalex.org/W4389162736","https://openalex.org/W4390017976"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2111241003","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W2355315220","https://openalex.org/W4200391368","https://openalex.org/W2210979487"],"abstract_inverted_index":{"Transformers":[0],"have":[1],"increasingly":[2],"become":[3],"the":[4,43,47,135,168],"backbone":[5],"of":[6,46,143,170],"modern":[7],"AI,":[8],"excelling":[9],"in":[10,50,109,127,198,226,231],"tasks":[11],"across":[12,181],"natural":[13],"language":[14],"processing":[15],"(NLP)":[16],"and":[17,34,80,131,140,146,172,205,213,228],"computer":[18],"vision":[19],"(CV).":[20],"However,":[21],"deploying":[22],"them":[23],"on":[24,41,193],"resource-constrained":[25],"platforms":[26],"is":[27],"challenging":[28],"due":[29],"to":[30,96,123,153],"their":[31],"high":[32,175],"computational":[33,44,99,159,189],"energy":[35,207,232],"demands.":[36],"Previous":[37],"efforts":[38],"primarily":[39],"focused":[40],"reducing":[42],"load":[45],"self-attention":[48,79,121],"module":[49,122],"Transformers,":[51],"often":[52],"neglecting":[53],"optimization":[54],"for":[55,120,150],"other":[56,236],"parts,":[57],"like":[58],"feed-forward":[59],"network":[60],"(FFN)":[61],"modules.":[62,82],"To":[63],"address":[64],"this":[65],"gap,":[66],"we":[67,138,217],"propose":[68],"FAS-Trans,":[69],"an":[70,85],"innovative":[71,86],"algorithm-architecture":[72],"co-design":[73],"accelerator":[74],"that":[75,185,219],"efficiently":[76],"optimizes":[77],"both":[78,128],"FFN":[81,136,158],"FAS-Trans":[83,186,200,220],"incorporates":[84],"approximate":[87,106],"prediction":[88,107,119],"mechanism":[89,102],"utilizing":[90],"shifted-adders,":[91],"which":[92],"pre-estimates":[93],"matrix":[94],"sparsity":[95,118,142,171],"further":[97],"reduce":[98],"loads.":[100],"This":[101],"also":[103],"enables":[104],"reusing":[105],"values":[108,151],"subsequent":[110],"exact":[111],"computations.":[112],"Moreover,":[113],"our":[114],"approach":[115],"includes":[116],"cross-stage":[117],"minimize":[124],"computations":[125],"involved":[126],"QKV":[129],"generation":[130],"attention":[132],"computing.":[133],"In":[134],"module,":[137],"predict":[139],"exploit":[141],"FC1":[144],"block":[145],"employ":[147],"low-precision":[148],"multipliers":[149],"close":[152],"zero,":[154],"significantly":[155],"cutting":[156],"down":[157],"overhead.":[160],"Our":[161],"dedicated":[162],"hardware":[163,176],"architecture":[164],"can":[165,201],"effectively":[166],"handle":[167],"irregularities":[169],"multi-precision,":[173],"ensuring":[174],"resource":[177],"utilization.":[178],"Comprehensive":[179],"evaluations":[180],"multiple":[182],"benchmarks":[183],"demonstrate":[184],"reduces":[187],"normalized":[188],"overhead":[190],"by":[191],"41.9%":[192],"average":[194],"with":[195,211,235],"1%":[196],"loss":[197],"accuracy.":[199],"achieve":[202],"2.39-27.98\u00d7":[203],"speedup":[204],"18.9--72.6\u00d7":[206],"efficiency":[208,233],"improvement":[209,225,230],"compared":[210,234],"CPU":[212],"GPU":[214],"acceleration.":[215],"Furthermore,":[216],"observe":[218],"achieves":[221],"2.15--3.80\u00d7":[222],"speedup,":[223],"1.87--19.89\u00d7":[224],"throughput":[227],"2.4--5.6\u00d7":[229],"FPGA-based":[237],"Transformer":[238],"accelerators.":[239]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}