{"id":"https://openalex.org/W4379115941","doi":"https://doi.org/10.23919/date56975.2023.10137086","title":"Benchmarking Large Language Models for Automated Verilog RTL Code Generation","display_name":"Benchmarking Large Language Models for Automated Verilog RTL Code Generation","publication_year":2023,"publication_date":"2023-04-01","ids":{"openalex":"https://openalex.org/W4379115941","doi":"https://doi.org/10.23919/date56975.2023.10137086"},"language":"en","primary_location":{"id":"doi:10.23919/date56975.2023.10137086","is_oa":false,"landing_page_url":"https://doi.org/10.23919/date56975.2023.10137086","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 Design, Automation & Test in Europe Conference & Exhibition (DATE)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031485134","display_name":"Shailja Thakur","orcid":"https://orcid.org/0000-0001-9590-5061"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shailja Thakur","raw_affiliation_strings":["New York University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063392410","display_name":"Baleegh Ahmad","orcid":"https://orcid.org/0000-0001-6854-3966"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Baleegh Ahmad","raw_affiliation_strings":["New York University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057245539","display_name":"Zhenxing Fan","orcid":null},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhenxing Fan","raw_affiliation_strings":["New York University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053148078","display_name":"Hammond Pearce","orcid":"https://orcid.org/0000-0002-3488-7004"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hammond Pearce","raw_affiliation_strings":["New York University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069388936","display_name":"Benjamin Tan","orcid":"https://orcid.org/0000-0002-7642-3638"},"institutions":[{"id":"https://openalex.org/I168635309","display_name":"University of Calgary","ror":"https://ror.org/03yjb2x39","country_code":"CA","type":"education","lineage":["https://openalex.org/I168635309"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Benjamin Tan","raw_affiliation_strings":["University of Calgary"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Calgary","institution_ids":["https://openalex.org/I168635309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059648257","display_name":"Ramesh Karri","orcid":"https://orcid.org/0000-0001-7989-5617"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ramesh Karri","raw_affiliation_strings":["New York University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060815601","display_name":"Brendan Dolan-Gavitt","orcid":"https://orcid.org/0000-0002-8867-4282"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Brendan Dolan-Gavitt","raw_affiliation_strings":["New York University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5010950688","display_name":"Siddharth Garg","orcid":"https://orcid.org/0000-0002-6158-9512"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Siddharth Garg","raw_affiliation_strings":["New York University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University","institution_ids":["https://openalex.org/I57206974"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":48.5931,"has_fulltext":false,"cited_by_count":150,"citation_normalized_percentile":{"value":0.9990816,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9911999702453613,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8393778800964355},{"id":"https://openalex.org/keywords/verilog","display_name":"Verilog","score":0.8180805444717407},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.6698789596557617},{"id":"https://openalex.org/keywords/compiler","display_name":"Compiler","score":0.6440329551696777},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.6399624347686768},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.5013821125030518},{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.48559778928756714},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.2532777786254883},{"id":"https://openalex.org/keywords/field-programmable-gate-array","display_name":"Field-programmable gate array","score":0.09244462847709656}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8393778800964355},{"id":"https://openalex.org/C2779030575","wikidata":"https://www.wikidata.org/wiki/Q827773","display_name":"Verilog","level":3,"score":0.8180805444717407},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.6698789596557617},{"id":"https://openalex.org/C169590947","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler","level":2,"score":0.6440329551696777},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6399624347686768},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5013821125030518},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.48559778928756714},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.2532777786254883},{"id":"https://openalex.org/C42935608","wikidata":"https://www.wikidata.org/wiki/Q190411","display_name":"Field-programmable gate array","level":2,"score":0.09244462847709656}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.23919/date56975.2023.10137086","is_oa":false,"landing_page_url":"https://doi.org/10.23919/date56975.2023.10137086","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 Design, Automation & Test in Europe Conference & Exhibition (DATE)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6100000143051147}],"awards":[{"id":"https://openalex.org/G2264616164","display_name":null,"funder_award_id":"1553419,1646671,2039607","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8605406651","display_name":null,"funder_award_id":"77191NC","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W46679369","https://openalex.org/W2099671082","https://openalex.org/W2345350761","https://openalex.org/W2618419749","https://openalex.org/W2896457183","https://openalex.org/W2965627384","https://openalex.org/W2973727699","https://openalex.org/W3082750925","https://openalex.org/W3118781290","https://openalex.org/W3177813494","https://openalex.org/W4288057765","https://openalex.org/W4292779060","https://openalex.org/W4385245566","https://openalex.org/W6601894380","https://openalex.org/W6739901393","https://openalex.org/W6766457457","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W1667647204","https://openalex.org/W2404647514","https://openalex.org/W4247536566","https://openalex.org/W4241418540","https://openalex.org/W2018477250","https://openalex.org/W3119814709","https://openalex.org/W1508895727","https://openalex.org/W4252501555","https://openalex.org/W2003848320","https://openalex.org/W2056593505"],"abstract_inverted_index":{"Automating":[0],"hardware":[1,24],"design":[2,30],"could":[3],"obviate":[4],"a":[5,22,38,96,140],"signif-icant":[6],"amount":[7],"of":[8,64,102,110,128],"human":[9],"error":[10],"from":[11,80],"the":[12,62,100,121,147],"engineering":[13],"process":[14],"and":[15,29,82,95,159],"lead":[16],"to":[17,27,49,66,98,108],"fewer":[18],"errors.":[19],"Verilog":[20,35,77,83,103],"is":[21,37],"popular":[23],"description":[25],"language":[26,44],"model":[28],"digital":[31],"systems,":[32],"thus":[33],"generating":[34],"code":[36,52,104,132],"critical":[39],"first":[40],"step.":[41],"Emerging":[42],"large":[43],"models":[45],"(LLMs)":[46],"are":[47],"able":[48],"write":[50],"high-quality":[51],"in":[53,106,124],"other":[54],"programming":[55],"languages.":[56],"In":[57],"this":[58],"paper,":[59],"we":[60,72],"characterize":[61],"ability":[63],"LLMs":[65,75,125],"generate":[67],"useful":[68],"Verilog.":[69],"For":[70],"this,":[71],"fine-tune":[73],"pre-trained":[74],"on":[76],"datasets":[78],"collected":[79],"GitHub":[81],"textbooks.":[84],"We":[85,154],"construct":[86],"an":[87],"evaluation":[88],"framework":[89],"comprising":[90],"test-benches":[91],"for":[92],"functional":[93,138],"analysis":[94],"flow":[97],"test":[99],"syntax":[101],"generated":[105],"response":[107],"problems":[109],"varying":[111],"difficulty.":[112],"Our":[113],"findings":[114],"show":[115],"that":[116],"across":[117],"our":[118,156],"problem":[119],"scenarios,":[120],"fine-tuning":[122],"results":[123],"more":[126],"capable":[127],"producing":[129],"syntactically":[130],"correct":[131],"(25.9%":[133],"overall).":[134,153],"Further,":[135],"when":[136],"analyzing":[137],"correctness,":[139],"fine-tuned":[141],"open-source":[142],"CodeGen":[143],"LLM":[144,151,160],"can":[145],"outperform":[146],"state-of-the-art":[148],"commercial":[149],"Codex":[150],"(6.5%":[152],"release":[155],"training/evaluation":[157],"scripts":[158],"checkpoints":[161],"as":[162],"open":[163],"source":[164],"contributions.":[165]},"counts_by_year":[{"year":2026,"cited_by_count":26},{"year":2025,"cited_by_count":69},{"year":2024,"cited_by_count":47},{"year":2023,"cited_by_count":8}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}