{"id":"https://openalex.org/W7135213754","doi":"https://doi.org/10.48550/arxiv.2603.11287","title":"Synthesis-in-the-Loop Evaluation of LLMs for RTL Generation: Quality, Reliability, and Failure Modes","display_name":"Synthesis-in-the-Loop Evaluation of LLMs for RTL Generation: Quality, Reliability, and Failure Modes","publication_year":2026,"publication_date":"2026-03-11","ids":{"openalex":"https://openalex.org/W7135213754","doi":"https://doi.org/10.48550/arxiv.2603.11287"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.11287","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.11287","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.11287","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128986457","display_name":"Weimin Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Weimin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100618877","display_name":"Zeng Wang","orcid":"https://orcid.org/0009-0002-5218-9856"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113123602","display_name":"Minghao Shao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shao, Minghao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121568580","display_name":"Ramesh Karri","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karri, Ramesh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128991912","display_name":"Muhammad Shafique","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shafique, Muhammad","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052751303","display_name":"Johann Knechtel","orcid":"https://orcid.org/0000-0001-5093-2939"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Knechtel, Johann","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059987567","display_name":"Ozgur Sinanoglu","orcid":"https://orcid.org/0000-0003-0782-0397"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sinanoglu, Ozgur","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129096604","display_name":"Xiaolong Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Xiaolong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10904","display_name":"Embedded Systems Design Techniques","score":0.8030999898910522,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10904","display_name":"Embedded Systems Design Techniques","score":0.8030999898910522,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.07039999961853027,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.015599999576807022,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.46939998865127563},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4611999988555908},{"id":"https://openalex.org/keywords/source-lines-of-code","display_name":"Source lines of code","score":0.4255000054836273},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.4052000045776367},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3950999975204468},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.361299991607666}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6062999963760376},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.46939998865127563},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4611999988555908},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.4255000054836273},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.4052000045776367},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3950999975204468},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.361299991607666},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.3203999996185303},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3156000077724457},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2930999994277954},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.28850001096725464},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.2874000072479248},{"id":"https://openalex.org/C2779030575","wikidata":"https://www.wikidata.org/wiki/Q827773","display_name":"Verilog","level":3,"score":0.28619998693466187},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.26080000400543213}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.11287","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.11287","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.11287","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.11287","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"score":0.4213395416736603,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"RTL":[0],"generation":[1],"is":[2],"more":[3],"than":[4],"code":[5],"synthesis.":[6],"Designs":[7],"must":[8],"be":[9],"syntactically":[10],"valid,":[11],"synthesizable,":[12],"correct,":[13],"hardware-efficient.":[14],"SOTA":[15],"evaluations":[16],"stop":[17],"at":[18,79],"functional":[19],"correctness":[20],"and":[21,26,41,54,82,98,126,139],"do":[22],"not":[23],"measure":[24],"synthesis":[25,114,127],"implementation":[27],"quality.":[28],"This":[29],"paper":[30],"evaluates":[31],"32":[32],"language":[33],"models":[34,71,86,120,130],"on":[35],"202":[36],"Verilog":[37],"tasks":[38],"from":[39],"VerilogEval":[40],"RTLLM":[42],"using":[43],"the":[44],"Hardware":[45],"Quality":[46],"Index":[47],"(HQI)":[48],"that":[49],"combines":[50],"post-synthesis":[51],"area,":[52],"delay,":[53],"warnings":[55],"related":[56],"to":[57,135],"expert":[58],"references":[59],"in":[60],"a":[61,142],"Nangate45":[62],"45\\,nm":[63],"flow.":[64],"Three":[65],"performance":[66],"regimes":[67],"emerge:":[68],"14":[69],"frontier":[70],"achieve":[72],"HQI":[73,103],"$>$":[74],"66,":[75],"led":[76],"by":[77],"Gemini-3-Pro":[78],"87.5\\%":[80],"coverage":[81],"85.1":[83],"HQI;":[84,89],"15":[85],"cluster":[87],"43--66":[88],"3":[90],"are":[91],"below":[92],"43.":[93],"Gap":[94],"between":[95],"best-of-five":[96],"capability":[97],"single-attempt":[99],"quality":[100],"spans":[101],"3.7--22.1":[102],"points,":[104],"limiting":[105],"integration":[106],"into":[107],"agentic":[108],"pipelines.":[109],"A":[110],"taxonomy":[111],"of":[112],"195":[113],"failures":[115],"reveals":[116],"systematic":[117],"divergence:":[118],"proprietary":[119],"fail":[121,131],"late":[122],"through":[123],"elaboration":[124],"errors":[125],"timeout;":[128],"open":[129],"early":[132],"often":[133],"due":[134],"missing":[136],"module":[137],"wrappers":[138],"non-synthesizable":[140],"constructs,":[141],"pattern":[143],"consistent":[144],"with":[145],"training":[146],"corpora":[147],"skewed":[148],"toward":[149],"simulation":[150],"over":[151],"synthesis-grade":[152],"RTL.":[153]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-03-14T00:00:00"}