{"id":"https://openalex.org/W4410583605","doi":"https://doi.org/10.23919/date64628.2025.10992720","title":"FVEval: Understanding Language Model Capabilities in Formal Verification of Digital Hardware","display_name":"FVEval: Understanding Language Model Capabilities in Formal Verification of Digital Hardware","publication_year":2025,"publication_date":"2025-03-31","ids":{"openalex":"https://openalex.org/W4410583605","doi":"https://doi.org/10.23919/date64628.2025.10992720"},"language":"en","primary_location":{"id":"doi:10.23919/date64628.2025.10992720","is_oa":false,"landing_page_url":"https://doi.org/10.23919/date64628.2025.10992720","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 Design, Automation & Test in Europe Conference (DATE)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5112909763","display_name":"Minwoo Kang","orcid":null},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Minwoo Kang","raw_affiliation_strings":["University of California,Berkeley,CA"],"affiliations":[{"raw_affiliation_string":"University of California,Berkeley,CA","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100681148","display_name":"Mingjie Liu","orcid":"https://orcid.org/0000-0002-7336-8282"},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mingjie Liu","raw_affiliation_strings":["NVIDIA,Santa Clara,CA"],"affiliations":[{"raw_affiliation_string":"NVIDIA,Santa Clara,CA","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071065752","display_name":"Ghaith Bany Hamad","orcid":"https://orcid.org/0000-0002-4354-2710"},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ghaith Bany Hamad","raw_affiliation_strings":["NVIDIA,Santa Clara,CA"],"affiliations":[{"raw_affiliation_string":"NVIDIA,Santa Clara,CA","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084835632","display_name":"Syed Suhaib","orcid":null},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Syed M. Suhaib","raw_affiliation_strings":["NVIDIA,Santa Clara,CA"],"affiliations":[{"raw_affiliation_string":"NVIDIA,Santa Clara,CA","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029928585","display_name":"Haoxing Ren","orcid":"https://orcid.org/0000-0003-1028-3860"},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haoxing Ren","raw_affiliation_strings":["NVIDIA,Santa Clara,CA"],"affiliations":[{"raw_affiliation_string":"NVIDIA,Santa Clara,CA","institution_ids":["https://openalex.org/I4210127875"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5112909763"],"corresponding_institution_ids":["https://openalex.org/I95457486"],"apc_list":null,"apc_paid":null,"fwci":11.5613,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.98478336,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10142","display_name":"Formal Methods in Verification","score":0.8867999911308289,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10142","display_name":"Formal Methods in Verification","score":0.8867999911308289,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.8568999767303467,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11450","display_name":"Model-Driven Software Engineering Techniques","score":0.7975999712944031,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7790361642837524},{"id":"https://openalex.org/keywords/formal-verification","display_name":"Formal verification","score":0.5867918133735657},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4382200241088867},{"id":"https://openalex.org/keywords/hardware-description-language","display_name":"Hardware description language","score":0.4128819704055786},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.32361721992492676},{"id":"https://openalex.org/keywords/computer-hardware","display_name":"Computer hardware","score":0.287309467792511},{"id":"https://openalex.org/keywords/field-programmable-gate-array","display_name":"Field-programmable gate array","score":0.14649125933647156}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7790361642837524},{"id":"https://openalex.org/C111498074","wikidata":"https://www.wikidata.org/wiki/Q173326","display_name":"Formal verification","level":2,"score":0.5867918133735657},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4382200241088867},{"id":"https://openalex.org/C42143788","wikidata":"https://www.wikidata.org/wiki/Q173341","display_name":"Hardware description language","level":3,"score":0.4128819704055786},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.32361721992492676},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.287309467792511},{"id":"https://openalex.org/C42935608","wikidata":"https://www.wikidata.org/wiki/Q190411","display_name":"Field-programmable gate array","level":2,"score":0.14649125933647156}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.23919/date64628.2025.10992720","is_oa":false,"landing_page_url":"https://doi.org/10.23919/date64628.2025.10992720","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 Design, Automation & Test in Europe Conference (DATE)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":13,"referenced_works":["https://openalex.org/W2946741689","https://openalex.org/W3082750925","https://openalex.org/W3123567287","https://openalex.org/W4387321091","https://openalex.org/W4388040387","https://openalex.org/W4388040405","https://openalex.org/W4389166695","https://openalex.org/W4389166737","https://openalex.org/W4391681217","https://openalex.org/W4399120235","https://openalex.org/W4401567960","https://openalex.org/W4403023462","https://openalex.org/W4404134046"],"related_works":["https://openalex.org/W1917929250","https://openalex.org/W2122949436","https://openalex.org/W2081964949","https://openalex.org/W78782492","https://openalex.org/W2587067507","https://openalex.org/W2077870657","https://openalex.org/W4312497944","https://openalex.org/W1547989843","https://openalex.org/W2165627905","https://openalex.org/W2132512458"],"abstract_inverted_index":{"The":[0,107],"remarkable":[1],"reasoning":[2,131],"and":[3,95,136,155,175,190,206],"code":[4,208],"generation":[5,121],"capabilities":[6,116],"of":[7,35,54,60,66,73,83,110,122,151,170],"large":[8],"language":[9,128],"models":[10,38],"(LLMs)":[11],"have":[12],"spurred":[13],"significant":[14,58],"interest":[15],"in":[16,23,102,201],"applying":[17,36],"LLMs":[18,188],"to":[19,39,45,105,130,157],"enable":[20,195],"task":[21],"automation":[22,68],"digital":[24,202],"chip":[25],"design.":[26],"In":[27,86],"particular,":[28],"recent":[29],"work":[30],"has":[31,77],"investigated":[32],"early":[33],"ideas":[34],"these":[37],"formal":[40],"verification":[41,153],"(FV),":[42],"an":[43],"approach":[44],"verifying":[46],"hardware":[47],"implementations":[48],"that":[49,113],"can":[50],"provide":[51],"strong":[52],"guarantees":[53],"confidence":[55],"but":[56],"demands":[57],"amounts":[59],"human":[61,142],"effort.":[62],"While":[63],"the":[64,81,91,120,133],"value":[65],"LLM-driven":[67],"is":[69,209],"evident,":[70],"our":[71],"understanding":[72],"model":[74],"performance,":[75],"however,":[76],"been":[78],"hindered":[79],"by":[80],"lack":[82],"holistic":[84],"evaluation.":[85],"response,":[87],"we":[88,147,184,192],"present":[89,148],"FVEval,":[90,180],"first":[92],"comprehensive":[93],"benchmark":[94,108,205],"evaluation":[96,207],"framework":[97],"for":[98],"characterizing":[99],"LLM":[100,115],"performance":[101],"tasks":[103],"pertaining":[104],"FV.":[106,203],"consists":[109],"three":[111],"sub-tasks":[112],"measure":[114],"at":[117,211],"different":[118],"levels-from":[119],"SystemVerilog":[123],"assertions":[124,138],"(SVA)":[125],"given":[126],"natural":[127],"descriptions":[129],"about":[132],"design":[134],"RTL":[135],"suggesting":[137],"directly":[139],"without":[140],"additional":[141],"input.":[143],"As":[144],"test":[145],"instances,":[146],"both":[149,173],"collections":[150],"expert-written":[152],"collateral":[154],"methodologies":[156],"scalably":[158],"generate":[159],"synthetic":[160],"examples":[161],"aligned":[162],"with":[163],"industrial":[164],"FV":[165],"workflows.":[166],"A":[167],"wide":[168],"range":[169],"existing":[171],"LLMs,":[172],"proprietary":[174],"open-source,":[176],"are":[177],"evaluated":[178],"against":[179],"based":[181],"on":[182],"which":[183],"investigate":[185],"where":[186],"today's":[187],"stand":[189],"how":[191],"might":[193],"further":[194],"their":[196],"application":[197],"toward":[198],"improving":[199],"productivity":[200],"Our":[204],"available":[210],"https://github.com/NVlabs/FVEval.":[212]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":5}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}