{"id":"https://openalex.org/W4415003693","doi":"https://doi.org/10.1109/mlcad65511.2025.11189228","title":"TuRTLe: A Unified Evaluation of LLMs for RTL Generation","display_name":"TuRTLe: A Unified Evaluation of LLMs for RTL Generation","publication_year":2025,"publication_date":"2025-09-08","ids":{"openalex":"https://openalex.org/W4415003693","doi":"https://doi.org/10.1109/mlcad65511.2025.11189228"},"language":"en","primary_location":{"id":"doi:10.1109/mlcad65511.2025.11189228","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mlcad65511.2025.11189228","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 ACM/IEEE 7th Symposium on Machine Learning for CAD (MLCAD)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.1109/MLCAD65511.2025.11189228","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010831226","display_name":"Dario Garc\u00eda-Gasulla","orcid":"https://orcid.org/0000-0001-6732-5641"},"institutions":[{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]},{"id":"https://openalex.org/I2799803557","display_name":"Barcelona Supercomputing Center","ror":"https://ror.org/05sd8tv96","country_code":"ES","type":"facility","lineage":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"Dario Garcia-Gasulla","raw_affiliation_strings":["Barcelona Supercomputing Center (BSC)"],"affiliations":[{"raw_affiliation_string":"Barcelona Supercomputing Center (BSC)","institution_ids":["https://openalex.org/I9617848","https://openalex.org/I2799803557"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048219922","display_name":"G\u00f6k\u00e7en Kestor","orcid":"https://orcid.org/0000-0002-9105-5634"},"institutions":[{"id":"https://openalex.org/I2799803557","display_name":"Barcelona Supercomputing Center","ror":"https://ror.org/05sd8tv96","country_code":"ES","type":"facility","lineage":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]},{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Gokcen Kestor","raw_affiliation_strings":["Barcelona Supercomputing Center (BSC)"],"affiliations":[{"raw_affiliation_string":"Barcelona Supercomputing Center (BSC)","institution_ids":["https://openalex.org/I9617848","https://openalex.org/I2799803557"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119925069","display_name":"Emanuele Parisi","orcid":null},"institutions":[{"id":"https://openalex.org/I2799803557","display_name":"Barcelona Supercomputing Center","ror":"https://ror.org/05sd8tv96","country_code":"ES","type":"facility","lineage":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]},{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Emanuele Parisi","raw_affiliation_strings":["Barcelona Supercomputing Center (BSC)"],"affiliations":[{"raw_affiliation_string":"Barcelona Supercomputing Center (BSC)","institution_ids":["https://openalex.org/I9617848","https://openalex.org/I2799803557"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119925070","display_name":"Miquel Albert\u00ed-Binimelis","orcid":null},"institutions":[{"id":"https://openalex.org/I2799803557","display_name":"Barcelona Supercomputing Center","ror":"https://ror.org/05sd8tv96","country_code":"ES","type":"facility","lineage":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]},{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Miquel Albert\u00ed-Binimelis","raw_affiliation_strings":["Barcelona Supercomputing Center (BSC)"],"affiliations":[{"raw_affiliation_string":"Barcelona Supercomputing Center (BSC)","institution_ids":["https://openalex.org/I9617848","https://openalex.org/I2799803557"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057603637","display_name":"Carlos Guti\u00e9rrez","orcid":"https://orcid.org/0000-0002-6234-7849"},"institutions":[{"id":"https://openalex.org/I2799803557","display_name":"Barcelona Supercomputing Center","ror":"https://ror.org/05sd8tv96","country_code":"ES","type":"facility","lineage":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]},{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Cristian Gutierrez","raw_affiliation_strings":["Barcelona Supercomputing Center (BSC)"],"affiliations":[{"raw_affiliation_string":"Barcelona Supercomputing Center (BSC)","institution_ids":["https://openalex.org/I9617848","https://openalex.org/I2799803557"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119925071","display_name":"Razine Moundir Ghorab","orcid":null},"institutions":[{"id":"https://openalex.org/I2799803557","display_name":"Barcelona Supercomputing Center","ror":"https://ror.org/05sd8tv96","country_code":"ES","type":"facility","lineage":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]},{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Razine Moundir Ghorab","raw_affiliation_strings":["Barcelona Supercomputing Center (BSC)"],"affiliations":[{"raw_affiliation_string":"Barcelona Supercomputing Center (BSC)","institution_ids":["https://openalex.org/I9617848","https://openalex.org/I2799803557"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119925072","display_name":"Orlando Montenegro","orcid":null},"institutions":[{"id":"https://openalex.org/I2799803557","display_name":"Barcelona Supercomputing Center","ror":"https://ror.org/05sd8tv96","country_code":"ES","type":"facility","lineage":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]},{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Orlando Montenegro","raw_affiliation_strings":["Barcelona Supercomputing Center (BSC)"],"affiliations":[{"raw_affiliation_string":"Barcelona Supercomputing Center (BSC)","institution_ids":["https://openalex.org/I9617848","https://openalex.org/I2799803557"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119925073","display_name":"Bernat Homs","orcid":null},"institutions":[{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]},{"id":"https://openalex.org/I2799803557","display_name":"Barcelona Supercomputing Center","ror":"https://ror.org/05sd8tv96","country_code":"ES","type":"facility","lineage":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Bernat Homs","raw_affiliation_strings":["Barcelona Supercomputing Center (BSC)"],"affiliations":[{"raw_affiliation_string":"Barcelona Supercomputing Center (BSC)","institution_ids":["https://openalex.org/I9617848","https://openalex.org/I2799803557"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5088729105","display_name":"Miquel Moret\u00f3","orcid":"https://orcid.org/0000-0002-9848-8758"},"institutions":[{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]},{"id":"https://openalex.org/I2799803557","display_name":"Barcelona Supercomputing Center","ror":"https://ror.org/05sd8tv96","country_code":"ES","type":"facility","lineage":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Miquel Moreto","raw_affiliation_strings":["Barcelona Supercomputing Center (BSC)"],"affiliations":[{"raw_affiliation_string":"Barcelona Supercomputing Center (BSC)","institution_ids":["https://openalex.org/I9617848","https://openalex.org/I2799803557"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5010831226"],"corresponding_institution_ids":["https://openalex.org/I2799803557","https://openalex.org/I9617848"],"apc_list":null,"apc_paid":null,"fwci":2.1186,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.9022086,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"12"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11195","display_name":"Simulation Techniques and Applications","score":0.8592000007629395,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11195","display_name":"Simulation Techniques and Applications","score":0.8592000007629395,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13286","display_name":"Modeling and Simulation Systems","score":0.8468000292778015,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12810","display_name":"Real-time simulation and control systems","score":0.7271000146865845,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4893999993801117},{"id":"https://openalex.org/keywords/syntax","display_name":"Syntax","score":0.4339999854564667},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4332999885082245},{"id":"https://openalex.org/keywords/automated-reasoning","display_name":"Automated reasoning","score":0.391400009393692},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.3668000102043152},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3312999904155731},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.32179999351501465}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6309999823570251},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4893999993801117},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4438999891281128},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.4339999854564667},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4332999885082245},{"id":"https://openalex.org/C195344581","wikidata":"https://www.wikidata.org/wiki/Q2555318","display_name":"Automated reasoning","level":2,"score":0.391400009393692},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3668000102043152},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3312999904155731},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3278000056743622},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.32179999351501465},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3181000053882599},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.30809998512268066},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.2962999939918518},{"id":"https://openalex.org/C64260653","wikidata":"https://www.wikidata.org/wiki/Q1194864","display_name":"Electronic design automation","level":2,"score":0.2896000146865845},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.2800000011920929},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.267300009727478},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.2635999917984009},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26010000705718994}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/mlcad65511.2025.11189228","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mlcad65511.2025.11189228","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 ACM/IEEE 7th Symposium on Machine Learning for CAD (MLCAD)","raw_type":"proceedings-article"},{"id":"pmh:oai:zenodo.org:17433164","is_oa":true,"landing_page_url":"https://doi.org/10.1109/MLCAD65511.2025.11189228","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"MLCAD, MLCAD Symposium 2025 - 7th ACM/IEEE International Symposium on Machine Learning for CAD, Chaminade Resort, Santa Cruz, California, 8-10 September 2025","raw_type":"info:eu-repo/semantics/conferencePaper"}],"best_oa_location":{"id":"pmh:oai:zenodo.org:17433164","is_oa":true,"landing_page_url":"https://doi.org/10.1109/MLCAD65511.2025.11189228","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"MLCAD, MLCAD Symposium 2025 - 7th ACM/IEEE International Symposium on Machine Learning for CAD, Chaminade Resort, Santa Cruz, California, 8-10 September 2025","raw_type":"info:eu-repo/semantics/conferencePaper"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Rapid":[0],"advancements":[1],"in":[2,11,77,121,150,167,175],"LLMs":[3,79,98,142],"have":[4],"driven":[5],"the":[6,19,111,156,188],"adoption":[7],"of":[8,21,32,118,139,199],"generative":[9],"AI":[10],"domains":[12],"like":[13],"Electronic":[14],"Design":[15],"Automation":[16],"(EDA).":[17],"Within":[18],"field":[20],"software":[22],"development,":[23],"EDA":[24],"presents":[25],"unique":[26],"challenges":[27],"derived":[28],"from":[29],"specific":[30,160],"requirements":[31,63],"generated":[33],"RTL":[34,36,81,101],"code;":[35],"code":[37],"must":[38],"not":[39],"only":[40],"be":[41],"syntactically":[42],"correct":[43],"and":[44,57,109,129,148],"functionally":[45],"accurate,":[46],"but":[47],"also":[48],"synthesizable":[49],"by":[50],"hardware":[51],"generators,":[52],"while":[53,178,201],"matching":[54],"performance,":[55],"power":[56],"area":[58],"(PPA)":[59],"constraints.":[60],"These":[61],"additional":[62],"introduce":[64],"complexities":[65],"that":[66,180],"existing":[67,107],"code-generation":[68],"benchmarks":[69,108],"often":[70],"fail":[71],"to":[72,95],"capture,":[73],"limiting":[74],"their":[75,146],"effectiveness":[76],"evaluating":[78],"for":[80,159,207],"generation.":[82],"To":[83],"address":[84],"this":[85,134],"gap,":[86],"we":[87],"propose":[88],"TURTLE,":[89],"a":[90,115,136],"unified":[91],"evaluation":[92,112],"framework":[93],"designed":[94],"systematically":[96],"assess":[97],"across":[99],"key":[100],"generation":[102],"tasks.":[103,152],"TURTLE":[104],"integrates":[105],"multiple":[106],"automates":[110],"process,":[113],"enabling":[114],"comprehensive":[116],"assessment":[117],"LLM":[119],"performance":[120],"syntax":[122,195],"correctness,":[123,125,196],"functional":[124],"synthesis,":[126],"PPA":[127],"optimization,":[128],"exact":[130],"line":[131],"completion.":[132],"Using":[133],"framework,":[135],"diverse":[137],"set":[138],"forty":[140],"open":[141,205],"are":[143,165,173],"assessed,":[144],"tracking":[145],"strengths":[147],"weaknesses":[149],"EDA-specific":[151],"Our":[153],"results":[154],"identify":[155],"best":[157,189],"match":[158],"tasks":[161],"(e.g.,":[162],"base":[163],"models":[164,172,182],"better":[166,174],"module":[168],"completion":[169],"tasks,":[170],"instruct-tuned":[171],"specification-to-RTL":[176],"tasks),":[177],"finding":[179],"recent":[181],"with":[183],"autoregressive":[184],"reasoning":[185],"chain":[186],"perform":[187],"overall.":[190],"Some":[191],"benchmarks,":[192],"particularly":[193],"within":[194],"show":[197],"signs":[198],"saturation,":[200],"others":[202],"remain":[203],"as":[204],"problems":[206],"LLMs.":[208]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-03-28T08:17:26.163206","created_date":"2025-10-10T00:00:00"}