{"id":"https://openalex.org/W4405656900","doi":"https://doi.org/10.1109/access.2026.3685519","title":"SimBench: A Framework for Evaluating and Diagnosing LLM-Based Digital-Twin Generation for Multi-Physics Simulation","display_name":"SimBench: A Framework for Evaluating and Diagnosing LLM-Based Digital-Twin Generation for Multi-Physics Simulation","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W4405656900","doi":"https://doi.org/10.1109/access.2026.3685519"},"language":"en","primary_location":{"id":"doi:10.1109/access.2026.3685519","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2026.3685519","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/access.2026.3685519","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5143220681","display_name":"Jingquan Wang","orcid":"https://orcid.org/0000-0001-8246-8018"},"institutions":[{"id":"https://openalex.org/I135310074","display_name":"University of Wisconsin\u2013Madison","ror":"https://ror.org/01y2jtd41","country_code":"US","type":"education","lineage":["https://openalex.org/I135310074"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jingquan Wang","raw_affiliation_strings":["University of Wisconsin–Madison, Madison, WI, USA"],"raw_orcid":"https://orcid.org/0000-0001-8246-8018","affiliations":[{"raw_affiliation_string":"University of Wisconsin–Madison, Madison, WI, USA","institution_ids":["https://openalex.org/I135310074"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119930519","display_name":"Andrew Negrut","orcid":null},"institutions":[{"id":"https://openalex.org/I74775410","display_name":"Rice University","ror":"https://ror.org/008zs3103","country_code":"US","type":"education","lineage":["https://openalex.org/I74775410"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew Negrut","raw_affiliation_strings":["Department of Computer Science, Rice University, Houston, TX, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Rice University, Houston, TX, USA","institution_ids":["https://openalex.org/I74775410"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139494398","display_name":"Hongyu Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I135310074","display_name":"University of Wisconsin\u2013Madison","ror":"https://ror.org/01y2jtd41","country_code":"US","type":"education","lineage":["https://openalex.org/I135310074"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hongyu Wang","raw_affiliation_strings":["University of Wisconsin–Madison, Madison, WI, USA"],"raw_orcid":"https://orcid.org/0009-0008-5478-5372","affiliations":[{"raw_affiliation_string":"University of Wisconsin–Madison, Madison, WI, USA","institution_ids":["https://openalex.org/I135310074"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103226976","display_name":"Harry Zhang","orcid":"https://orcid.org/0009-0003-1903-274X"},"institutions":[{"id":"https://openalex.org/I135310074","display_name":"University of Wisconsin\u2013Madison","ror":"https://ror.org/01y2jtd41","country_code":"US","type":"education","lineage":["https://openalex.org/I135310074"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Harry Zhang","raw_affiliation_strings":["University of Wisconsin–Madison, Madison, WI, USA"],"raw_orcid":"https://orcid.org/0009-0003-1903-274X","affiliations":[{"raw_affiliation_string":"University of Wisconsin–Madison, Madison, WI, USA","institution_ids":["https://openalex.org/I135310074"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027224030","display_name":"Dan Negru\u0163","orcid":"https://orcid.org/0000-0003-1565-2784"},"institutions":[{"id":"https://openalex.org/I135310074","display_name":"University of Wisconsin\u2013Madison","ror":"https://ror.org/01y2jtd41","country_code":"US","type":"education","lineage":["https://openalex.org/I135310074"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dan Negrut","raw_affiliation_strings":["University of Wisconsin–Madison, Madison, WI, USA"],"raw_orcid":"https://orcid.org/0000-0003-1565-2784","affiliations":[{"raw_affiliation_string":"University of Wisconsin–Madison, Madison, WI, USA","institution_ids":["https://openalex.org/I135310074"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.00394623,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"14","issue":null,"first_page":"61784","last_page":"61808"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10763","display_name":"Digital Transformation in Industry","score":0.7871999740600586,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10763","display_name":"Digital Transformation in Industry","score":0.7871999740600586,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.7838000059127808,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.7732999920845032,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7649132013320923},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.538161039352417},{"id":"https://openalex.org/keywords/turn","display_name":"Turn (biochemistry)","score":0.48993057012557983},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3576006293296814},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.12611830234527588},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.08190777897834778},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.06874516606330872}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7649132013320923},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.538161039352417},{"id":"https://openalex.org/C85641259","wikidata":"https://www.wikidata.org/wiki/Q290042","display_name":"Turn (biochemistry)","level":2,"score":0.48993057012557983},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3576006293296814},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.12611830234527588},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.08190777897834778},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.06874516606330872},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1109/access.2026.3685519","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2026.3685519","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2408.11987","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.11987","pdf_url":"https://arxiv.org/pdf/2408.11987","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"pmh:doi:10.48550/arxiv.2408.11987","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"pmh:oai:doaj.org/article:0080beeebd5d412e8b16a005effbd362","is_oa":true,"landing_page_url":"https://doaj.org/article/0080beeebd5d412e8b16a005effbd362","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 14, Pp 61784-61808 (2026)","raw_type":"article"},{"id":"doi:10.48550/arxiv.2408.11987","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2408.11987","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1109/access.2026.3685519","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2026.3685519","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G939003905","display_name":null,"funder_award_id":"CMMI2153855","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W4231070408","https://openalex.org/W943151747","https://openalex.org/W2048981943","https://openalex.org/W2130974462"],"abstract_inverted_index":{"We":[0,45],"introduce":[1],"SimBench,":[2],"a":[3,29,84,95],"benchmark":[4,34],"designed":[5],"to":[6,38,41,72,94,117,123,127,159],"evaluate":[7],"the":[8,76,80,99,108,114,124,153],"proficiency":[9],"of":[10,31,155],"simulator-oriented":[11],"LLMs":[12],"(S-LLMs)":[13],"in":[14,23,105,121],"generating":[15],"digital":[16,129,161],"twins":[17,130,162],"(DTs)":[18],"that":[19,64],"can":[20],"be":[21],"used":[22,116],"simulators":[24],"for":[25,75,131,163],"virtual":[26],"testing.":[27],"Given":[28],"collection":[30],"S-LLMs,":[32],"this":[33,47],"ranks":[35],"them":[36],"according":[37],"their":[39],"ability":[40,126,158],"produce":[42],"high-quality":[43],"DTs.":[44],"demonstrate":[46],"by":[48,79],"comparing":[49],"over":[50],"33":[51],"open-":[52],"and":[53,69,86,97,141,151,173],"closed-source":[54],"S-LLMs.":[55],"Using":[56],"multi-turn":[57],"interactions,":[58],"SimBench":[59],"employs":[60],"an":[61,119,156],"LLM-as-a-judge":[62],"(J-LLM)":[63],"leverages":[65],"both":[66],"predefined":[67],"rules":[68],"human-in-the-loop":[70],"guidance":[71],"assign":[73],"scores":[74],"DTs":[77],"generated":[78],"S-LLM,":[81],"thus":[82],"providing":[83],"consistent":[85],"expert-inspired":[87],"evaluation":[88],"protocol.":[89],"The":[90,144],"J-LLM":[91],"is":[92,103,148],"specific":[93],"simulator,":[96],"herein":[98],"proposed":[100,145],"benchmarking":[101,146],"approach":[102],"demonstrated":[104],"conjunction":[106],"with":[107],"open-sourceChrono":[109],"multi-physics":[110],"simulator.":[111],"Chrono":[112],"provided":[113],"backdrop":[115],"assess":[118],"S-LLM":[120],"relation":[122],"latter\u2019s":[125],"create":[128],"multibody":[132],"dynamics,":[133,138,140],"finite":[134],"element":[135],"analysis,":[136],"vehicle":[137],"robotic":[139],"sensor":[142],"simulations.":[143],"principle":[147],"broadly":[149],"applicable":[150],"enables":[152],"assessment":[154],"S-LLM\u2019s":[157],"generate":[160],"other":[164],"simulation":[165],"packages,":[166],"e.g.,":[167],"ANSYS,":[168],"ABAQUS,":[169],"OpenFOAM,":[170],"StarCCM+,":[171],"IsaacSim,":[172],"pyBullet.":[174]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}