{"id":"https://openalex.org/W4402568829","doi":"https://doi.org/10.1109/tcsii.2024.3462560","title":"An FPGA-Based Transformer Accelerator With Parallel Unstructured Sparsity Handling for Question-Answering Applications","display_name":"An FPGA-Based Transformer Accelerator With Parallel Unstructured Sparsity Handling for Question-Answering Applications","publication_year":2024,"publication_date":"2024-09-17","ids":{"openalex":"https://openalex.org/W4402568829","doi":"https://doi.org/10.1109/tcsii.2024.3462560"},"language":"en","primary_location":{"id":"doi:10.1109/tcsii.2024.3462560","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsii.2024.3462560","pdf_url":null,"source":{"id":"https://openalex.org/S93916849","display_name":"IEEE Transactions on Circuits & Systems II Express Briefs","issn_l":"1549-7747","issn":["1549-7747","1558-3791"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems II: Express Briefs","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085590234","display_name":"Rujian Cao","orcid":"https://orcid.org/0000-0003-0612-2941"},"institutions":[{"id":"https://openalex.org/I204512498","display_name":"University of Macau","ror":"https://ror.org/01r4q9n85","country_code":"MO","type":"education","lineage":["https://openalex.org/I204512498"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN","MO"],"is_corresponding":true,"raw_author_name":"Rujian Cao","raw_affiliation_strings":["State-Key Laboratory of Analog and Mixed-Signal VLSI, Institute of Microelectronics, and the Faculty of Science and Technology - ECE, University of Macau, Macau, China","State-Key Laboratory of Analog and Mixed-Signal VLSI/ Institute of Microelectronics, and Faculty of Science and Technology -ECE, University of Macau, Macao, China"],"raw_orcid":"https://orcid.org/0000-0003-0612-2941","affiliations":[{"raw_affiliation_string":"State-Key Laboratory of Analog and Mixed-Signal VLSI, Institute of Microelectronics, and the Faculty of Science and Technology - ECE, University of Macau, Macau, China","institution_ids":["https://openalex.org/I204512498"]},{"raw_affiliation_string":"State-Key Laboratory of Analog and Mixed-Signal VLSI/ Institute of Microelectronics, and Faculty of Science and Technology -ECE, University of Macau, Macao, China","institution_ids":["https://openalex.org/I204512498","https://openalex.org/I4210119392"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102865728","display_name":"Zhongyu Zhao","orcid":"https://orcid.org/0000-0002-5221-6490"},"institutions":[{"id":"https://openalex.org/I204512498","display_name":"University of Macau","ror":"https://ror.org/01r4q9n85","country_code":"MO","type":"education","lineage":["https://openalex.org/I204512498"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN","MO"],"is_corresponding":false,"raw_author_name":"Zhongyu Zhao","raw_affiliation_strings":["State-Key Laboratory of Analog and Mixed-Signal VLSI, Institute of Microelectronics, and the Faculty of Science and Technology - ECE, University of Macau, Macau, China","State-Key Laboratory of Analog and Mixed-Signal VLSI/ Institute of Microelectronics, and Faculty of Science and Technology -ECE, University of Macau, Macao, China"],"raw_orcid":"https://orcid.org/0000-0002-5221-6490","affiliations":[{"raw_affiliation_string":"State-Key Laboratory of Analog and Mixed-Signal VLSI, Institute of Microelectronics, and the Faculty of Science and Technology - ECE, University of Macau, Macau, China","institution_ids":["https://openalex.org/I204512498"]},{"raw_affiliation_string":"State-Key Laboratory of Analog and Mixed-Signal VLSI/ Institute of Microelectronics, and Faculty of Science and Technology -ECE, University of Macau, Macao, China","institution_ids":["https://openalex.org/I204512498","https://openalex.org/I4210119392"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038010873","display_name":"Ka-Fai Un","orcid":"https://orcid.org/0000-0002-7574-4755"},"institutions":[{"id":"https://openalex.org/I204512498","display_name":"University of Macau","ror":"https://ror.org/01r4q9n85","country_code":"MO","type":"education","lineage":["https://openalex.org/I204512498"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN","MO"],"is_corresponding":false,"raw_author_name":"Ka-Fai Un","raw_affiliation_strings":["State-Key Laboratory of Analog and Mixed-Signal VLSI, Institute of Microelectronics, and the Faculty of Science and Technology - ECE, University of Macau, Macau, China","State-Key Laboratory of Analog and Mixed-Signal VLSI/ Institute of Microelectronics, and Faculty of Science and Technology -ECE, University of Macau, Macao, China"],"raw_orcid":"https://orcid.org/0000-0002-7574-4755","affiliations":[{"raw_affiliation_string":"State-Key Laboratory of Analog and Mixed-Signal VLSI, Institute of Microelectronics, and the Faculty of Science and Technology - ECE, University of Macau, Macau, China","institution_ids":["https://openalex.org/I204512498"]},{"raw_affiliation_string":"State-Key Laboratory of Analog and Mixed-Signal VLSI/ Institute of Microelectronics, and Faculty of Science and Technology -ECE, University of Macau, Macao, China","institution_ids":["https://openalex.org/I204512498","https://openalex.org/I4210119392"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090222712","display_name":"Wei-Han Yu","orcid":"https://orcid.org/0000-0002-9079-5227"},"institutions":[{"id":"https://openalex.org/I204512498","display_name":"University of Macau","ror":"https://ror.org/01r4q9n85","country_code":"MO","type":"education","lineage":["https://openalex.org/I204512498"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN","MO"],"is_corresponding":false,"raw_author_name":"Wei-Han Yu","raw_affiliation_strings":["State-Key Laboratory of Analog and Mixed-Signal VLSI, Institute of Microelectronics, and the Faculty of Science and Technology - ECE, University of Macau, Macau, China","State-Key Laboratory of Analog and Mixed-Signal VLSI/ Institute of Microelectronics, and Faculty of Science and Technology -ECE, University of Macau, Macao, China"],"raw_orcid":"https://orcid.org/0000-0002-9079-5227","affiliations":[{"raw_affiliation_string":"State-Key Laboratory of Analog and Mixed-Signal VLSI, Institute of Microelectronics, and the Faculty of Science and Technology - ECE, University of Macau, Macau, China","institution_ids":["https://openalex.org/I204512498"]},{"raw_affiliation_string":"State-Key Laboratory of Analog and Mixed-Signal VLSI/ Institute of Microelectronics, and Faculty of Science and Technology -ECE, University of Macau, Macao, China","institution_ids":["https://openalex.org/I204512498","https://openalex.org/I4210119392"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106943352","display_name":"Rui P. Martins","orcid":"https://orcid.org/0000-0003-2821-648X"},"institutions":[{"id":"https://openalex.org/I204512498","display_name":"University of Macau","ror":"https://ror.org/01r4q9n85","country_code":"MO","type":"education","lineage":["https://openalex.org/I204512498"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN","MO"],"is_corresponding":false,"raw_author_name":"Rui P. Martins","raw_affiliation_strings":["State-Key Laboratory of Analog and Mixed-Signal VLSI, Institute of Microelectronics, and the Faculty of Science and Technology - ECE, University of Macau, Macau, China","State-Key Laboratory of Analog and Mixed-Signal VLSI/ Institute of Microelectronics, and Faculty of Science and Technology -ECE, University of Macau, Macao, China"],"raw_orcid":"https://orcid.org/0000-0003-2821-648X","affiliations":[{"raw_affiliation_string":"State-Key Laboratory of Analog and Mixed-Signal VLSI, Institute of Microelectronics, and the Faculty of Science and Technology - ECE, University of Macau, Macau, China","institution_ids":["https://openalex.org/I204512498"]},{"raw_affiliation_string":"State-Key Laboratory of Analog and Mixed-Signal VLSI/ Institute of Microelectronics, and Faculty of Science and Technology -ECE, University of Macau, Macao, China","institution_ids":["https://openalex.org/I204512498","https://openalex.org/I4210119392"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5058845450","display_name":"Pui\u2010In Mak","orcid":"https://orcid.org/0000-0002-3579-8740"},"institutions":[{"id":"https://openalex.org/I204512498","display_name":"University of Macau","ror":"https://ror.org/01r4q9n85","country_code":"MO","type":"education","lineage":["https://openalex.org/I204512498"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN","MO"],"is_corresponding":false,"raw_author_name":"Pui-In Mak","raw_affiliation_strings":["State-Key Laboratory of Analog and Mixed-Signal VLSI, Institute of Microelectronics, and the Faculty of Science and Technology - ECE, University of Macau, Macau, China","State-Key Laboratory of Analog and Mixed-Signal VLSI/ Institute of Microelectronics, and Faculty of Science and Technology -ECE, University of Macau, Macao, China"],"raw_orcid":"https://orcid.org/0000-0002-3579-8740","affiliations":[{"raw_affiliation_string":"State-Key Laboratory of Analog and Mixed-Signal VLSI, Institute of Microelectronics, and the Faculty of Science and Technology - ECE, University of Macau, Macau, China","institution_ids":["https://openalex.org/I204512498"]},{"raw_affiliation_string":"State-Key Laboratory of Analog and Mixed-Signal VLSI/ Institute of Microelectronics, and Faculty of Science and Technology -ECE, University of Macau, Macao, China","institution_ids":["https://openalex.org/I204512498","https://openalex.org/I4210119392"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5085590234"],"corresponding_institution_ids":["https://openalex.org/I204512498","https://openalex.org/I4210119392"],"apc_list":null,"apc_paid":null,"fwci":0.9228,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.78486749,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"71","issue":"11","first_page":"4688","last_page":"4692"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11283","display_name":"Experimental Learning in Engineering","score":0.9531999826431274,"subfield":{"id":"https://openalex.org/subfields/2214","display_name":"Media Technology"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11283","display_name":"Experimental Learning in Engineering","score":0.9531999826431274,"subfield":{"id":"https://openalex.org/subfields/2214","display_name":"Media Technology"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10904","display_name":"Embedded Systems Design Techniques","score":0.9052000045776367,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.9035000205039978,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/field-programmable-gate-array","display_name":"Field-programmable gate array","score":0.6584120988845825},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6323036551475525},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6024022102355957},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.5261508822441101},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.3873053193092346},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.3284015655517578},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.298447847366333},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.17903771996498108},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.17105171084403992},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.15509819984436035}],"concepts":[{"id":"https://openalex.org/C42935608","wikidata":"https://www.wikidata.org/wiki/Q190411","display_name":"Field-programmable gate array","level":2,"score":0.6584120988845825},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6323036551475525},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6024022102355957},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.5261508822441101},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3873053193092346},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.3284015655517578},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.298447847366333},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.17903771996498108},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.17105171084403992},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.15509819984436035},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcsii.2024.3462560","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsii.2024.3462560","pdf_url":null,"source":{"id":"https://openalex.org/S93916849","display_name":"IEEE Transactions on Circuits & Systems II Express Briefs","issn_l":"1549-7747","issn":["1549-7747","1558-3791"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems II: Express Briefs","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2299472110","display_name":null,"funder_award_id":"0160/2022/A","funder_id":"https://openalex.org/F4320315893","funder_display_name":"VNU Science and Technology Development Fund"},{"id":"https://openalex.org/G6265068319","display_name":null,"funder_award_id":"MYRG-GRG2023-00012-IME","funder_id":"https://openalex.org/F4320322841","funder_display_name":"Universidade de Macau"},{"id":"https://openalex.org/G6548532224","display_name":null,"funder_award_id":"004/2023/SKL","funder_id":"https://openalex.org/F4320315893","funder_display_name":"VNU Science and Technology Development Fund"}],"funders":[{"id":"https://openalex.org/F4320315893","display_name":"VNU Science and Technology Development Fund","ror":"https://ror.org/044vr6g03"},{"id":"https://openalex.org/F4320322841","display_name":"Universidade de Macau","ror":"https://ror.org/01r4q9n85"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W2889787757","https://openalex.org/W2965373594","https://openalex.org/W3011411500","https://openalex.org/W3034288893","https://openalex.org/W3047848469","https://openalex.org/W3090912412","https://openalex.org/W3104415840","https://openalex.org/W3145926676","https://openalex.org/W3159727696","https://openalex.org/W3162090017","https://openalex.org/W3162542754","https://openalex.org/W3164217046","https://openalex.org/W4212888381","https://openalex.org/W4285117917","https://openalex.org/W4289656095","https://openalex.org/W4291653242","https://openalex.org/W4291653336","https://openalex.org/W4364302069","https://openalex.org/W4389888389","https://openalex.org/W4389934542","https://openalex.org/W6763701032","https://openalex.org/W6810370753"],"related_works":["https://openalex.org/W2384605597","https://openalex.org/W2387743295","https://openalex.org/W2115758952","https://openalex.org/W4381058564","https://openalex.org/W3003945460","https://openalex.org/W2964413124","https://openalex.org/W4288267738","https://openalex.org/W1967938402","https://openalex.org/W2386041993","https://openalex.org/W1608572506"],"abstract_inverted_index":{"Dataflow":[0],"management":[1],"provides":[2],"limited":[3],"performance":[4,29],"improvement":[5],"to":[6,11,30,50,64,107],"the":[7,17,31,40,44,80,86,90,96,110,114,122],"transformer":[8,33],"model":[9],"due":[10],"its":[12],"lesser":[13],"weight":[14,102],"reuse":[15],"than":[16],"convolution":[18],"neural":[19],"network.":[20],"The":[21],"cosFormer":[22,45],"reduced":[23],"computational":[24],"complexity":[25],"while":[26],"achieving":[27],"comparable":[28],"vanilla":[32],"for":[34,138],"natural":[35],"language":[36],"processing":[37],"tasks.":[38],"However,":[39],"unstructured":[41,59,73,101],"sparsity":[42,60,74,77,112],"in":[43,95],"makes":[46],"it":[47],"a":[48,57,126],"challenge":[49],"be":[51],"implemented":[52],"efficiently.":[53,70],"This":[54],"brief":[55],"proposes":[56],"parallel":[58],"handling":[61],"(PUSH)":[62],"scheme":[63],"compute":[65],"sparse-dense":[66],"matrix":[67],"multiplication":[68],"(SDMM)":[69],"It":[71],"transforms":[72],"into":[75],"structured":[76,111],"and":[78,92,131],"reduces":[79],"total":[81],"memory":[82,87],"access":[83],"by":[84],"balancing":[85],"accesses":[88],"of":[89,113,128,135],"sparse":[91],"dense":[93],"matrices":[94],"SDMM.":[97],"We":[98],"also":[99],"employ":[100],"pruning":[103],"cooperating":[104],"with":[105,141],"PUSH":[106],"further":[108],"increase":[109],"model.":[115],"Through":[116],"verification":[117],"on":[118],"an":[119,132],"FPGA":[120],"platform,":[121],"proposed":[123],"accelerator":[124],"achieves":[125],"throughput":[127],"2.82":[129],"TOPS":[130],"energy":[133],"efficiency":[134],"144.8":[136],"GOPs/W":[137],"HotpotQA":[139],"dataset":[140],"long":[142],"sequences.":[143]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}