{"id":"https://openalex.org/W4389166737","doi":"https://doi.org/10.1109/iccad57390.2023.10323812","title":"Invited Paper: VerilogEval: Evaluating Large Language Models for Verilog Code Generation","display_name":"Invited Paper: VerilogEval: Evaluating Large Language Models for Verilog Code Generation","publication_year":2023,"publication_date":"2023-10-28","ids":{"openalex":"https://openalex.org/W4389166737","doi":"https://doi.org/10.1109/iccad57390.2023.10323812"},"language":"en","primary_location":{"id":"doi:10.1109/iccad57390.2023.10323812","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccad57390.2023.10323812","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE/ACM International Conference on Computer Aided Design (ICCAD)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100681148","display_name":"Mingjie Liu","orcid":"https://orcid.org/0000-0002-7336-8282"},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Mingjie Liu","raw_affiliation_strings":["NVIDIA Corporation"],"affiliations":[{"raw_affiliation_string":"NVIDIA Corporation","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050270997","display_name":"Nathaniel Pinckney","orcid":"https://orcid.org/0000-0001-6159-8964"},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nathaniel Pinckney","raw_affiliation_strings":["NVIDIA Corporation"],"affiliations":[{"raw_affiliation_string":"NVIDIA Corporation","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010156116","display_name":"Brucek Khailany","orcid":"https://orcid.org/0000-0002-7584-3489"},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Brucek Khailany","raw_affiliation_strings":["NVIDIA Corporation"],"affiliations":[{"raw_affiliation_string":"NVIDIA Corporation","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029928585","display_name":"Haoxing Ren","orcid":"https://orcid.org/0000-0003-1028-3860"},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haoxing Ren","raw_affiliation_strings":["NVIDIA Corporation"],"affiliations":[{"raw_affiliation_string":"NVIDIA Corporation","institution_ids":["https://openalex.org/I4210127875"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5100681148"],"corresponding_institution_ids":["https://openalex.org/I4210127875"],"apc_list":null,"apc_paid":null,"fwci":23.6756,"has_fulltext":false,"cited_by_count":136,"citation_normalized_percentile":{"value":0.99692069,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9900000095367432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8584355115890503},{"id":"https://openalex.org/keywords/verilog","display_name":"Verilog","score":0.769690215587616},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.6616312861442566},{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.5688747763633728},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.5535816550254822},{"id":"https://openalex.org/keywords/hardware-description-language","display_name":"Hardware description language","score":0.5414519309997559},{"id":"https://openalex.org/keywords/netlist","display_name":"Netlist","score":0.5207345485687256},{"id":"https://openalex.org/keywords/compiler","display_name":"Compiler","score":0.5158835649490356},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5105122327804565},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5083789229393005},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.48886317014694214},{"id":"https://openalex.org/keywords/bootstrapping","display_name":"Bootstrapping (finance)","score":0.48712894320487976},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.46917086839675903},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.41165703535079956},{"id":"https://openalex.org/keywords/field-programmable-gate-array","display_name":"Field-programmable gate array","score":0.24788448214530945},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.2461622655391693},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.1076374351978302},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.08927297592163086}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8584355115890503},{"id":"https://openalex.org/C2779030575","wikidata":"https://www.wikidata.org/wiki/Q827773","display_name":"Verilog","level":3,"score":0.769690215587616},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6616312861442566},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.5688747763633728},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.5535816550254822},{"id":"https://openalex.org/C42143788","wikidata":"https://www.wikidata.org/wiki/Q173341","display_name":"Hardware description language","level":3,"score":0.5414519309997559},{"id":"https://openalex.org/C177650935","wikidata":"https://www.wikidata.org/wiki/Q1760303","display_name":"Netlist","level":2,"score":0.5207345485687256},{"id":"https://openalex.org/C169590947","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler","level":2,"score":0.5158835649490356},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5105122327804565},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5083789229393005},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.48886317014694214},{"id":"https://openalex.org/C207609745","wikidata":"https://www.wikidata.org/wiki/Q4944086","display_name":"Bootstrapping (finance)","level":2,"score":0.48712894320487976},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.46917086839675903},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.41165703535079956},{"id":"https://openalex.org/C42935608","wikidata":"https://www.wikidata.org/wiki/Q190411","display_name":"Field-programmable gate array","level":2,"score":0.24788448214530945},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.2461622655391693},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.1076374351978302},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.08927297592163086},{"id":"https://openalex.org/C106159729","wikidata":"https://www.wikidata.org/wiki/Q2294553","display_name":"Financial economics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/iccad57390.2023.10323812","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccad57390.2023.10323812","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE/ACM International Conference on Computer Aided Design (ICCAD)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.46000000834465027}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W347636100","https://openalex.org/W2101105183","https://openalex.org/W2132069633","https://openalex.org/W2938704169","https://openalex.org/W2970575144","https://openalex.org/W3104578551","https://openalex.org/W3118781290","https://openalex.org/W3126675481","https://openalex.org/W3161457214","https://openalex.org/W3177765786","https://openalex.org/W3177813494","https://openalex.org/W4223908421","https://openalex.org/W4226278401","https://openalex.org/W4226485558","https://openalex.org/W4286987939","https://openalex.org/W4287024925","https://openalex.org/W4292779060","https://openalex.org/W4320005767","https://openalex.org/W4360836968","https://openalex.org/W4361866125","https://openalex.org/W4367000491","https://openalex.org/W4378508556","https://openalex.org/W4379115941","https://openalex.org/W4380993527","https://openalex.org/W4381586770","https://openalex.org/W4384345639","https://openalex.org/W4385572634","https://openalex.org/W4388040405","https://openalex.org/W4391558462","https://openalex.org/W4393145520","https://openalex.org/W6761551260","https://openalex.org/W6764109406","https://openalex.org/W6778883912","https://openalex.org/W6788175385","https://openalex.org/W6790588633","https://openalex.org/W6794686226","https://openalex.org/W6798182279","https://openalex.org/W6800166007","https://openalex.org/W6800875267","https://openalex.org/W6809583738","https://openalex.org/W6810738896","https://openalex.org/W6810874553","https://openalex.org/W6849941170","https://openalex.org/W6850820320","https://openalex.org/W6850936240","https://openalex.org/W6852043138","https://openalex.org/W6852952922","https://openalex.org/W6853517571","https://openalex.org/W6854013397","https://openalex.org/W6855837758"],"related_works":["https://openalex.org/W2004826944","https://openalex.org/W1935030095","https://openalex.org/W2136295006","https://openalex.org/W54894869","https://openalex.org/W2165480138","https://openalex.org/W2068696707","https://openalex.org/W2107240870","https://openalex.org/W2162678295","https://openalex.org/W2107517480","https://openalex.org/W2544421437"],"abstract_inverted_index":{"The":[0,58,81],"increasing":[1],"popularity":[2],"of":[3,33,49,62,66,98,115],"large":[4],"language":[5,117],"models":[6,118],"(LLMs)":[7],"has":[8],"paved":[9],"the":[10,31,53,94,99,110],"way":[11],"for":[12,26,37,89],"their":[13],"application":[14],"in":[15,30],"diverse":[16,64],"domains.":[17],"This":[18],"paper":[19],"proposes":[20],"a":[21,44,63,103],"benchmarking":[22],"framework":[23],"tailored":[24],"specifically":[25],"evaluating":[27],"LLM":[28,128],"performance":[29],"context":[32],"Verilog":[34,54,67,82,111],"code":[35,68,83,112],"generation":[36,69,113],"hardware":[38],"design":[39,101],"and":[40],"verification.":[41],"We":[42,106],"present":[43],"comprehensive":[45],"evaluation":[46,59],"dataset":[47],"consisting":[48],"156":[50],"problems":[51],"from":[52,72],"instructional":[55],"website":[56],"HDLBits.":[57],"set":[60,65],"consists":[61],"tasks,":[70],"ranging":[71],"simple":[73],"combinational":[74],"circuits":[75],"to":[76],"complex":[77],"finite":[78],"state":[79],"machines.":[80],"completions":[84],"can":[85],"be":[86,120],"automatically":[87],"tested":[88],"functional":[90],"correctness":[91],"by":[92,125],"comparing":[93],"transient":[95],"simulation":[96],"outputs":[97],"generated":[100,129],"with":[102,122,127],"golden":[104],"solution.":[105],"also":[107],"demonstrate":[108],"that":[109],"capability":[114],"pretrained":[116],"could":[119],"improved":[121],"supervised":[123],"fine-tuning":[124],"bootstrapping":[126],"synthetic":[130],"problem-code":[131],"pairs.":[132]},"counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":88},{"year":2024,"cited_by_count":36},{"year":2023,"cited_by_count":2}],"updated_date":"2026-03-28T08:17:26.163206","created_date":"2025-10-10T00:00:00"}