{"id":"https://openalex.org/W4393145520","doi":"https://doi.org/10.1109/asp-dac58780.2024.10473904","title":"RTLLM: An Open-Source Benchmark for Design RTL Generation with Large Language Model","display_name":"RTLLM: An Open-Source Benchmark for Design RTL Generation with Large Language Model","publication_year":2024,"publication_date":"2024-01-22","ids":{"openalex":"https://openalex.org/W4393145520","doi":"https://doi.org/10.1109/asp-dac58780.2024.10473904"},"language":"en","primary_location":{"id":"doi:10.1109/asp-dac58780.2024.10473904","is_oa":false,"landing_page_url":"https://doi.org/10.1109/asp-dac58780.2024.10473904","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 29th Asia and South Pacific Design Automation Conference (ASP-DAC)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101746530","display_name":"Yao Lu","orcid":"https://orcid.org/0009-0007-3230-7786"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yao Lu","raw_affiliation_strings":["Hong Kong University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101520717","display_name":"Shang Liu","orcid":"https://orcid.org/0009-0000-0057-7844"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shang Liu","raw_affiliation_strings":["Hong Kong University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100726176","display_name":"Qijun Zhang","orcid":"https://orcid.org/0009-0008-4749-2141"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Qijun Zhang","raw_affiliation_strings":["Hong Kong University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101509934","display_name":"Zhiyao Xie","orcid":"https://orcid.org/0009-0005-4608-6895"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhiyao Xie","raw_affiliation_strings":["Hong Kong University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology","institution_ids":["https://openalex.org/I200769079"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I200769079"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":148,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"722","last_page":"727"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.7286999821662903,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.7286999821662903,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.664900004863739,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7927465438842773},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6933531761169434},{"id":"https://openalex.org/keywords/open-source","display_name":"Open source","score":0.5211774706840515},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4681342542171478},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.44701117277145386},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.33386605978012085},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.3239385485649109},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.1011374294757843}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7927465438842773},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6933531761169434},{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.5211774706840515},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4681342542171478},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.44701117277145386},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33386605978012085},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3239385485649109},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.1011374294757843},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/asp-dac58780.2024.10473904","is_oa":false,"landing_page_url":"https://doi.org/10.1109/asp-dac58780.2024.10473904","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 29th Asia and South Pacific Design Automation Conference (ASP-DAC)","raw_type":"proceedings-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-136467","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-136467","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W2896457183","https://openalex.org/W3169517138","https://openalex.org/W3210573375","https://openalex.org/W4226485558","https://openalex.org/W4376167329","https://openalex.org/W4378508556","https://openalex.org/W4379115941","https://openalex.org/W4388040405","https://openalex.org/W6755207826","https://openalex.org/W6810874553","https://openalex.org/W6852746770","https://openalex.org/W6852952922"],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W2745001401","https://openalex.org/W4321353415","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W972276598","https://openalex.org/W2087343574","https://openalex.org/W4246352526","https://openalex.org/W2121910908"],"abstract_inverted_index":{"Inspired":[0],"by":[1,51],"the":[2,16,52,72,77,105,154],"recent":[3],"success":[4],"of":[5,18,80,131,156],"large":[6],"language":[7,100],"models":[8],"(LLMs)":[9],"like":[10],"ChatGPT,":[11],"researchers":[12],"start":[13],"to":[14,151],"explore":[15],"adoption":[17],"LLMs":[19],"for":[20,94],"agile":[21],"hardware":[22],"design,":[23],"such":[24],"as":[25],"generating":[26,95],"design":[27,73,78,82,96,107,120],"RTL":[28,97],"based":[29],"on":[30,71],"natural-language":[31],"instructions.":[32,101],"However,":[33],"in":[34,45,158],"existing":[35],"works,":[36],"their":[37],"target":[38],"designs":[39],"are":[40],"all":[41],"relatively":[42],"simple":[43],"and":[44,49,119],"a":[46,56,128],"small":[47],"scale,":[48],"proposed":[50,160],"authors":[53],"themselves,":[54],"making":[55],"fair":[57],"comparison":[58],"among":[59],"different":[60],"LLM":[61],"solutions":[62],"challenging.":[63],"In":[64,84],"addition,":[65],"many":[66],"prior":[67],"works":[68],"only":[69],"focus":[70],"correctness,":[74],"without":[75],"evaluating":[76],"qualities":[79],"generated":[81],"RTL.":[83],"this":[85],"work,":[86],"we":[87,109,137],"propose":[88,138],"an":[89,139],"open-source":[90],"benchmark":[91,124],"named":[92,114,147],"RTLLM,":[93],"with":[98],"natural":[99],"To":[102],"systematically":[103],"evaluate":[104],"auto-generated":[106],"RTL,":[108],"summarized":[110],"three":[111],"progressive":[112],"goals,":[113],"syntax":[115],"goal,":[116,118],"functionality":[117],"quality":[121],"goal.":[122],"This":[123],"can":[125],"automatically":[126],"provide":[127],"quantitative":[129],"evaluation":[130],"any":[132],"given":[133],"LLM-based":[134],"solution.":[135],"Furthermore,":[136],"easy-to-use":[140],"yet":[141],"surprisingly":[142],"effective":[143],"prompt":[144],"engineering":[145],"technique":[146],"self-planning,":[148],"which":[149],"proves":[150],"significantly":[152],"boost":[153],"performance":[155],"GPT-3.5":[157],"our":[159],"benchmark.":[161]},"counts_by_year":[{"year":2026,"cited_by_count":32},{"year":2025,"cited_by_count":77},{"year":2024,"cited_by_count":37},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-25T15:57:00.446498","created_date":"2025-10-10T00:00:00"}