{"id":"https://openalex.org/W4409285540","doi":"https://doi.org/10.1145/3676536.3676761","title":"ChatOPU: An FPGA-based Overlay Processor for Large Language Models with Unstructured Sparsity","display_name":"ChatOPU: An FPGA-based Overlay Processor for Large Language Models with Unstructured Sparsity","publication_year":2024,"publication_date":"2024-10-27","ids":{"openalex":"https://openalex.org/W4409285540","doi":"https://doi.org/10.1145/3676536.3676761"},"language":"en","primary_location":{"id":"doi:10.1145/3676536.3676761","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3676536.3676761","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3676536.3676761","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd IEEE/ACM International Conference on Computer-Aided Design","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3676536.3676761","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073130132","display_name":"Tiandong Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tiandong Zhao","raw_affiliation_strings":["University of California, Los Angeles, Los Angeles, California, United States"],"raw_orcid":"https://orcid.org/0009-0000-5348-1715","affiliations":[{"raw_affiliation_string":"University of California, Los Angeles, Los Angeles, California, United States","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101262539","display_name":"Shaoqiang Lu","orcid":null},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4394709157","display_name":"Eastern Institute of Technology, Ningbo","ror":"https://ror.org/036mbz113","country_code":null,"type":"education","lineage":["https://openalex.org/I4394709157"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaoqiang Lu","raw_affiliation_strings":["Ningbo Institute of Digital Twin, Eastern Institute of Technology, Ningbo, China","Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0002-8624-8666","affiliations":[{"raw_affiliation_string":"Ningbo Institute of Digital Twin, Eastern Institute of Technology, Ningbo, China","institution_ids":["https://openalex.org/I4394709157"]},{"raw_affiliation_string":"Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101491215","display_name":"C.-L. Wu","orcid":"https://orcid.org/0000-0002-9939-8922"},"institutions":[{"id":"https://openalex.org/I13591777","display_name":"University of Nottingham Ningbo China","ror":"https://ror.org/03y4dt428","country_code":"CN","type":"education","lineage":["https://openalex.org/I13591777","https://openalex.org/I142263535"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen Wu","raw_affiliation_strings":["Ningbo Institute of Digital Twin, Ningbo, China"],"raw_orcid":"https://orcid.org/0000-0002-9939-8922","affiliations":[{"raw_affiliation_string":"Ningbo Institute of Digital Twin, Ningbo, China","institution_ids":["https://openalex.org/I13591777"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008695429","display_name":"Lei He","orcid":"https://orcid.org/0000-0002-5266-3805"},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]},{"id":"https://openalex.org/I4394709157","display_name":"Eastern Institute of Technology, Ningbo","ror":"https://ror.org/036mbz113","country_code":null,"type":"education","lineage":["https://openalex.org/I4394709157"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lei He","raw_affiliation_strings":["Ningbo Institute of Digital Twin, Eastern Institute of Technology, Ningbo, China","University of California, Los Angeles, Los Angeles, California, USA"],"raw_orcid":"https://orcid.org/0000-0002-5266-3805","affiliations":[{"raw_affiliation_string":"Ningbo Institute of Digital Twin, Eastern Institute of Technology, Ningbo, China","institution_ids":["https://openalex.org/I4394709157"]},{"raw_affiliation_string":"University of California, Los Angeles, Los Angeles, California, USA","institution_ids":["https://openalex.org/I161318765"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.4437,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":{"value":0.90881528,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9890000224113464,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/overlay","display_name":"Overlay","score":0.8326718807220459},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8325023651123047},{"id":"https://openalex.org/keywords/field-programmable-gate-array","display_name":"Field-programmable gate array","score":0.7717442512512207},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5060686469078064},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.49774864315986633},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.42174625396728516},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.38454669713974},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3443566560745239},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.18982243537902832}],"concepts":[{"id":"https://openalex.org/C136085584","wikidata":"https://www.wikidata.org/wiki/Q910289","display_name":"Overlay","level":2,"score":0.8326718807220459},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8325023651123047},{"id":"https://openalex.org/C42935608","wikidata":"https://www.wikidata.org/wiki/Q190411","display_name":"Field-programmable gate array","level":2,"score":0.7717442512512207},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5060686469078064},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.49774864315986633},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.42174625396728516},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.38454669713974},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3443566560745239},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18982243537902832}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3676536.3676761","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3676536.3676761","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3676536.3676761","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd IEEE/ACM International Conference on Computer-Aided Design","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3676536.3676761","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3676536.3676761","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3676536.3676761","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd IEEE/ACM International Conference on Computer-Aided Design","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4409285540.pdf","grobid_xml":"https://content.openalex.org/works/W4409285540.grobid-xml"},"referenced_works_count":15,"referenced_works":["https://openalex.org/W2606722458","https://openalex.org/W2979455536","https://openalex.org/W3008408165","https://openalex.org/W3040024858","https://openalex.org/W3047848469","https://openalex.org/W3159727696","https://openalex.org/W4226439885","https://openalex.org/W4291653336","https://openalex.org/W4308083513","https://openalex.org/W4318256712","https://openalex.org/W4321637192","https://openalex.org/W4380881077","https://openalex.org/W4387321091","https://openalex.org/W4393147854","https://openalex.org/W4393578753"],"related_works":["https://openalex.org/W595346907","https://openalex.org/W153296825","https://openalex.org/W598989511","https://openalex.org/W2375779923","https://openalex.org/W2041986468","https://openalex.org/W1967800214","https://openalex.org/W2055675609","https://openalex.org/W1967938402","https://openalex.org/W2386041993","https://openalex.org/W1608572506"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,22],"(LLMs)":[3],"have":[4,23],"achieved":[5],"notable":[6],"success":[7],"on":[8,20,96,153,163,169],"many":[9],"applications":[10],"with":[11,85,135,171],"increasingly":[12],"tremendous":[13],"parameters":[14,122],"and":[15,34,48,108,117,141,159,167,177],"computations.":[16],"While":[17],"hardware":[18,68],"accelerators":[19,162],"Transformer-based":[21],"been":[24],"extensively":[25],"studied,":[26],"recent":[27],"work":[28,37],"mainly":[29],"assumes":[30],"structured":[31],"model":[32,83],"pruning":[33,84],"does":[35],"not":[36],"well":[38],"for":[39,78,105,119],"unstructured":[40,65,82,172],"sparsity":[41,173],"that":[42,55,151],"could":[43],"lead":[44],"to":[45,59,80,100,123],"more":[46],"parameter":[47],"computation":[49],"reduction.":[50],"The":[51],"reason":[52],"behind":[53],"is":[54,57],"it":[56],"difficult":[58],"exploit":[60],"data":[61,87,103],"reuse":[62,104],"from":[63],"the":[64,120,131,145],"sparsity,":[66],"leaving":[67],"underutilized.":[69],"This":[70],"paper":[71],"proposes":[72],"ChatOPU,":[73],"an":[74],"FPGA-based":[75,161],"overlay":[76],"processor":[77],"LLMs,":[79],"support":[81],"better":[86],"reuse.":[88],"First,":[89],"we":[90,113,129],"propose":[91],"a":[92,97],"new":[93],"diagonal":[94],"dataflow":[95],"systolic":[98],"array":[99],"obtain":[101],"efficient":[102,115],"both":[106],"sparse":[107,121],"dense":[109],"matrix":[110],"multiplication.":[111],"Second,":[112],"develop":[114],"encoding":[116],"decoding":[118],"save":[124],"off-chip":[125,132],"memory":[126],"traffic.":[127],"Moreover,":[128],"boost":[130],"bandwidth":[133],"utilization":[134],"pinned":[136],"on-chip":[137],"KV":[138],"cache":[139],"allocation":[140],"coalesced":[142],"access":[143],"throughout":[144],"LLM":[146],"inference.":[147],"Experimental":[148],"results":[149],"show":[150],"ChatOPU":[152],"Xilinx":[154],"U200":[155],"FPGA":[156],"outperforms":[157],"GPU":[158],"other":[160],"token/s":[164],"by":[165],"2.29\u00d7":[166],"1.63\u00d7":[168],"LLMs":[170],"across":[174],"different":[175],"input":[176],"output":[178],"sequence":[179],"lengths.":[180]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":6}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}