{"id":"https://openalex.org/W7160821619","doi":"https://doi.org/10.48550/arxiv.2605.07247","title":"EnvSimBench: A Benchmark for Evaluating and Improving LLM-Based Environment Simulation","display_name":"EnvSimBench: A Benchmark for Evaluating and Improving LLM-Based Environment Simulation","publication_year":2026,"publication_date":"2026-05-08","ids":{"openalex":"https://openalex.org/W7160821619","doi":"https://doi.org/10.48550/arxiv.2605.07247"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.07247","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.07247","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.07247","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135904864","display_name":"Yi Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065626573","display_name":"Tingfeng Hui","orcid":"https://orcid.org/0009-0006-3524-4904"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hui, TingFeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135835351","display_name":"Wei Zhang","orcid":"https://orcid.org/0009-0004-3200-3246"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135831080","display_name":"Li Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125105132","display_name":"Ningxin Su","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Su, Ningxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135861930","display_name":"Jian Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135834298","display_name":"Sen Su","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Su, Sen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.2290000021457672,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.2290000021457672,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.08869999647140503,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.06700000166893005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6262999773025513},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5390999913215637},{"id":"https://openalex.org/keywords/undo","display_name":"Undo","score":0.46050000190734863},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.4447999894618988},{"id":"https://openalex.org/keywords/flexibility","display_name":"Flexibility (engineering)","score":0.4417000114917755},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.3986999988555908},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.3862000107765198},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.3783999979496002},{"id":"https://openalex.org/keywords/path","display_name":"Path (computing)","score":0.37599998712539673},{"id":"https://openalex.org/keywords/unavailability","display_name":"Unavailability","score":0.3711000084877014}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.722100019454956},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6262999773025513},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5390999913215637},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.46860000491142273},{"id":"https://openalex.org/C2780154230","wikidata":"https://www.wikidata.org/wiki/Q513420","display_name":"Undo","level":2,"score":0.46050000190734863},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.4447999894618988},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.4417000114917755},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.3986999988555908},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.3862000107765198},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.3783999979496002},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3774999976158142},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.37599998712539673},{"id":"https://openalex.org/C2780505938","wikidata":"https://www.wikidata.org/wiki/Q17093282","display_name":"Unavailability","level":2,"score":0.3711000084877014},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36890000104904175},{"id":"https://openalex.org/C9354725","wikidata":"https://www.wikidata.org/wiki/Q286017","display_name":"Operationalization","level":2,"score":0.3562999963760376},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.3391000032424927},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.33379998803138733},{"id":"https://openalex.org/C63000827","wikidata":"https://www.wikidata.org/wiki/Q3080428","display_name":"Software portability","level":2,"score":0.32820001244544983},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.3257000148296356},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.31450000405311584},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.30090001225471497},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.29989999532699585},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.28940001130104065},{"id":"https://openalex.org/C130440534","wikidata":"https://www.wikidata.org/wiki/Q14946528","display_name":"Conflation","level":2,"score":0.2815999984741211},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.2800000011920929},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.2703999876976013},{"id":"https://openalex.org/C62989814","wikidata":"https://www.wikidata.org/wiki/Q854648","display_name":"Gossip","level":2,"score":0.26159998774528503},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.25699999928474426},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.25609999895095825},{"id":"https://openalex.org/C3739613","wikidata":"https://www.wikidata.org/wiki/Q679003","display_name":"Distributed Computing Environment","level":2,"score":0.2535000145435333},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.2522999942302704},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.2517000138759613}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.07247","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.07247","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.07247","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.07247","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/15","score":0.530807375907898,"display_name":"Life in Land"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Scalable":[0],"AI":[1],"agents":[2],"training":[3],"relies":[4],"on":[5,47,164],"interactive":[6],"environments":[7,18,39,61],"that":[8,72,82,148,201],"faithfully":[9],"simulate":[10,55],"the":[11,79,83,102,167],"consequences":[12],"of":[13,108],"agent":[14,74,240],"actions.":[15],"Manually":[16],"crafted":[17,38],"are":[19,245],"expensive":[20],"to":[21,24,35,87],"build,":[22],"brittle":[23],"extend,":[25],"and":[26,67,77,106,137,211,225,243],"fundamentally":[27],"limited":[28],"in":[29],"diversity.":[30],"A":[31],"promising":[32],"direction":[33],"is":[34],"replace":[36],"manually":[37],"with":[40,96,134],"LLM-simulated":[41,60],"counterparts.":[42],"However,":[43],"this":[44,91],"paradigm":[45,84],"hinges":[46],"an":[48],"unexamined":[49],"core":[50],"assumption:":[51],"LLMs":[52],"can":[53],"accurately":[54],"environmental":[56],"feedback.":[57],"In":[58],"practice,":[59],"suffer":[62,153],"from":[63,154],"hallucinations,":[64],"logical":[65],"inconsistencies,":[66],"silent":[68],"state":[69,157,169],"drift":[70],"failures":[71],"corrupt":[73],"reward":[75],"signals":[76],"compound":[78],"construction":[80],"costs":[81,213],"was":[85],"designed":[86],"eliminate.":[88],"To":[89],"address":[90],"gap,":[92],"we":[93],"propose":[94],"EnvSimBench":[95,218],"four":[97],"contributions:":[98],"1)":[99],"We":[100,120,195],"provide":[101],"first":[103],"formal":[104],"definition":[105],"operationalization":[107],"Environment":[109],"Simulation":[110],"Ability":[111,185],"(EnvSim":[112],"Ability)":[113],"as":[114,186,220],"a":[115,123,155,187,197,222,226,236],"quantifiable":[116],"research":[117],"objective.":[118],"2)":[119],"construct":[121],"EnvSimBench,":[122],"rigorous":[124],"benchmark":[125],"covering":[126],"400":[127],"samples":[128],"across":[129],"167":[130],"diverse":[131],"environments,":[132],"equipped":[133],"verifiable":[135],"labels":[136],"fine-grained":[138],"difficulty":[139],"stratification":[140],"along":[141],"three":[142],"axes.":[143],"3)":[144],"Systematic":[145],"evaluations":[146],"reveal":[147],"all":[149],"state-of-the-art":[150],"language":[151],"models":[152],"universal":[156],"change":[158],"cliff:":[159],"they":[160],"achieve":[161],"near-perfect":[162],"accuracy":[163],"tasks":[165],"when":[166,175],"environment":[168,206,233],"remains":[170],"invariant,":[171],"yet":[172,189],"fail":[173],"catastrophically":[174],"multiple":[176],"states":[177],"need":[178],"simultaneous":[179],"updates.":[180],"This":[181],"finding":[182],"exposes":[183],"EnvSim":[184],"critical":[188],"largely":[190],"unaddressed":[191],"capability":[192],"gap.":[193],"4)":[194],"design":[196],"constraint-driven":[198],"simulation":[199],"pipeline":[200],"substantially":[202],"reduces":[203],"hallucination,":[204],"boosts":[205],"synthesis":[207],"yield":[208],"by":[209,214],"6.8%,":[210],"cuts":[212],"over":[215],"90%.":[216],"Overall,":[217],"serves":[219],"both":[221],"diagnostic":[223],"framework":[224],"practical":[227],"optimization":[228],"path":[229],"for":[230,238],"reliable":[231],"LLM-based":[232],"simulation,":[234],"establishing":[235],"foundation":[237],"scalable":[239],"training.":[241],"Code":[242],"data":[244],"available":[246],"at":[247],"https://github.com/cookieApril/EnvSimBench":[248]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-12T00:00:00"}