{"id":"https://openalex.org/W4408844959","doi":"https://doi.org/10.1145/3723876","title":"Automatically Improving LLM-based Verilog Generation using EDA Tool Feedback","display_name":"Automatically Improving LLM-based Verilog Generation using EDA Tool Feedback","publication_year":2025,"publication_date":"2025-03-26","ids":{"openalex":"https://openalex.org/W4408844959","doi":"https://doi.org/10.1145/3723876"},"language":"en","primary_location":{"id":"doi:10.1145/3723876","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3723876","pdf_url":null,"source":{"id":"https://openalex.org/S105046310","display_name":"ACM Transactions on Design Automation of Electronic Systems","issn_l":"1084-4309","issn":["1084-4309","1557-7309"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Design Automation of Electronic Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024964362","display_name":"Jason Blocklove","orcid":"https://orcid.org/0009-0005-5619-4654"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jason Blocklove","raw_affiliation_strings":["New York University Tandon School of Engineering","Electrical and Computer Engineering, New York University Tandon School of Engineering, Brooklyn, United States"],"raw_orcid":"https://orcid.org/0009-0005-5619-4654","affiliations":[{"raw_affiliation_string":"New York University Tandon School of Engineering","institution_ids":["https://openalex.org/I57206974"]},{"raw_affiliation_string":"Electrical and Computer Engineering, New York University Tandon School of Engineering, Brooklyn, United States","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031485134","display_name":"Shailja Thakur","orcid":"https://orcid.org/0000-0001-9590-5061"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shailja Thakur","raw_affiliation_strings":["New York University Tandon School of Engineering","Center for cybersecurity, New York University Tandon School of Engineering, Brooklyn, United States"],"raw_orcid":"https://orcid.org/0000-0001-9590-5061","affiliations":[{"raw_affiliation_string":"New York University Tandon School of Engineering","institution_ids":["https://openalex.org/I57206974"]},{"raw_affiliation_string":"Center for cybersecurity, New York University Tandon School of Engineering, Brooklyn, United States","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069388936","display_name":"Benjamin Tan","orcid":"https://orcid.org/0000-0002-7642-3638"},"institutions":[{"id":"https://openalex.org/I168635309","display_name":"University of Calgary","ror":"https://ror.org/03yjb2x39","country_code":"CA","type":"education","lineage":["https://openalex.org/I168635309"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Benjamin Tan","raw_affiliation_strings":["Electrical and Software Engineering, University of Calgary","Electrical and Software Engineering, University of Calgary, Calgary, Canada"],"raw_orcid":"https://orcid.org/0000-0002-7642-3638","affiliations":[{"raw_affiliation_string":"Electrical and Software Engineering, University of Calgary","institution_ids":["https://openalex.org/I168635309"]},{"raw_affiliation_string":"Electrical and Software Engineering, University of Calgary, Calgary, Canada","institution_ids":["https://openalex.org/I168635309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053148078","display_name":"Hammond Pearce","orcid":"https://orcid.org/0000-0002-3488-7004"},"institutions":[{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Hammond Pearce","raw_affiliation_strings":["University of New South Wales Faculty of Engineering","University of New South Wales Faculty of Engineering,  Sydney, Australia"],"raw_orcid":"https://orcid.org/0000-0002-3488-7004","affiliations":[{"raw_affiliation_string":"University of New South Wales Faculty of Engineering","institution_ids":["https://openalex.org/I31746571"]},{"raw_affiliation_string":"University of New South Wales Faculty of Engineering,  Sydney, Australia","institution_ids":["https://openalex.org/I31746571"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010950688","display_name":"Siddharth Garg","orcid":"https://orcid.org/0000-0002-6158-9512"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Siddharth Garg","raw_affiliation_strings":["New York University Tandon School of Engineering","New York University Tandon School of Engineering,  New York, United States"],"raw_orcid":"https://orcid.org/0000-0002-6158-9512","affiliations":[{"raw_affiliation_string":"New York University Tandon School of Engineering","institution_ids":["https://openalex.org/I57206974"]},{"raw_affiliation_string":"New York University Tandon School of Engineering,  New York, United States","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5059648257","display_name":"Ramesh Karri","orcid":"https://orcid.org/0000-0001-7989-5617"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ramesh Karri","raw_affiliation_strings":["New York University Tandon School of Engineering","ECE, New York University Tandon School of Engineering, Brooklyn, United States"],"raw_orcid":"https://orcid.org/0000-0001-7989-5617","affiliations":[{"raw_affiliation_string":"New York University Tandon School of Engineering","institution_ids":["https://openalex.org/I57206974"]},{"raw_affiliation_string":"ECE, New York University Tandon School of Engineering, Brooklyn, United States","institution_ids":["https://openalex.org/I57206974"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":22.2935,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.99296565,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"30","issue":"6","first_page":"1","last_page":"26"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9876000285148621,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9876000285148621,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9837999939918518,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.9236882925033569},{"id":"https://openalex.org/keywords/verilog","display_name":"Verilog","score":0.7445940971374512},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.4866541028022766},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.41021835803985596},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.3459509015083313},{"id":"https://openalex.org/keywords/field-programmable-gate-array","display_name":"Field-programmable gate array","score":0.29180240631103516}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.9236882925033569},{"id":"https://openalex.org/C2779030575","wikidata":"https://www.wikidata.org/wiki/Q827773","display_name":"Verilog","level":3,"score":0.7445940971374512},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.4866541028022766},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.41021835803985596},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3459509015083313},{"id":"https://openalex.org/C42935608","wikidata":"https://www.wikidata.org/wiki/Q190411","display_name":"Field-programmable gate array","level":2,"score":0.29180240631103516}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3723876","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3723876","pdf_url":null,"source":{"id":"https://openalex.org/S105046310","display_name":"ACM Transactions on Design Automation of Electronic Systems","issn_l":"1084-4309","issn":["1084-4309","1557-7309"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Design Automation of Electronic Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":21,"referenced_works":["https://openalex.org/W3082750925","https://openalex.org/W3177813494","https://openalex.org/W4378508556","https://openalex.org/W4379115941","https://openalex.org/W4385774800","https://openalex.org/W4386794762","https://openalex.org/W4388040387","https://openalex.org/W4388040405","https://openalex.org/W4388275102","https://openalex.org/W4388555899","https://openalex.org/W4389166691","https://openalex.org/W4389814505","https://openalex.org/W4391631329","https://openalex.org/W4391631389","https://openalex.org/W4392405447","https://openalex.org/W4392544408","https://openalex.org/W4396786227","https://openalex.org/W4399354280","https://openalex.org/W4399695736","https://openalex.org/W4402952710","https://openalex.org/W4406071699"],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W1761969858","https://openalex.org/W2158571935","https://openalex.org/W2391854357","https://openalex.org/W2362026881","https://openalex.org/W3114194214","https://openalex.org/W1748531671","https://openalex.org/W4206948312","https://openalex.org/W2132512458"],"abstract_inverted_index":{"Traditionally,":[0],"digital":[1],"hardware":[2,9],"designs":[3,193],"are":[4,31],"written":[5],"in":[6,51,98,188,198,220,240],"the":[7,52,82,119,134,141,172,180,189,201,213,216],"Verilog":[8,122],"description":[10],"language":[11],"(HDL)":[12],"and":[13,22,57,124,129],"debugged":[14],"manually":[15],"by":[16,244],"engineers.":[17],"This":[18],"can":[19],"be":[20,162],"time-consuming":[21],"error-prone":[23],"for":[24],"complex":[25,175],"designs.":[26],"Large":[27],"Language":[28],"Models":[29],"(LLMs)":[30],"emerging":[32],"as":[33,67,222,225],"a":[34,59,73,185,195],"potential":[35],"tool":[36,158],"to":[37,86,95,126,161,236],"help":[38],"generate":[39,128],"fully":[40],"functioning":[41],"HDL":[42],"code,":[43],"but":[44,230],"most":[45,173],"works":[46],"have":[47],"focused":[48],"on":[49,152],"generation":[50],"single-shot":[53],"capacity:":[54],"i.e.,":[55],"run":[56],"evaluate,":[58],"process":[60],"that":[61],"does":[62,69],"not":[63,70],"leverage":[64,87,140],"debugging":[65],"and,":[66],"such,":[68],"adequately":[71],"reflect":[72],"realistic":[74],"development":[75],"process.":[76],"In":[77,179],"this":[78,209],"work,":[79],"we":[80,106,139,177,183],"evaluate":[81,146],"ability":[83],"of":[84,136,191,215],"LLMs":[85,117,138],"feedback":[88,159,217],"from":[89,121],"electronic":[90],"design":[91],"automation":[92],"(EDA)":[93],"tools":[94],"fix":[96],"mistakes":[97],"their":[99],"own":[100],"generated":[101],"Verilog.":[102,131],"To":[103,132],"accomplish":[104],"this,":[105],"present":[107],"an":[108,237],"open-source,":[109],"highly":[110],"customizable":[111],"framework,":[112],"AutoChip,":[113],"which":[114],"combines":[115],"conversational":[116,149],"with":[118,170,194,208,226],"output":[120],"compilers":[123],"simulations":[125],"iteratively":[127],"repair":[130],"determine":[133],"success":[135,224],"these":[137],"VerilogEval":[142],"benchmark":[143],"set.":[144],"We":[145],"four":[147],"state-of-the-art":[148],"LLMs,":[150],"focusing":[151],"readily":[153],"accessible":[154],"commercial":[155],"models.":[156],"EDA":[157],"proved":[160],"consistently":[163],"more":[164],"effective":[165],"than":[166],"zero-shot":[167,203,243],"prompting":[168],"only":[169],"GPT-4o,":[171],"computationally":[174],"model":[176,211],"evaluated.":[178],"best":[181,202],"case,":[182],"observed":[184],"5.8%":[186],"increase":[187],"number":[190],"successful":[192],"34.2%":[196],"decrease":[197,239],"cost":[199,234,241],"over":[200,242],"results.":[204],"Mixing":[205],"smaller":[206],"models":[207],"larger":[210],"at":[212],"end":[214],"iterations":[218],"resulted":[219],"equally":[221],"much":[223],"GPT-4o":[227],"using":[228],"feedback,":[229],"incurred":[231],"41.9%":[232],"lower":[233],"(corresponding":[235],"overall":[238],"89.6%).":[245]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":6}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}