{"id":"https://openalex.org/W7164837075","doi":"https://doi.org/10.48550/arxiv.2606.14574","title":"SIMMER: Benchmarking Latent Failures in LLM Executable Planning with a World Model","display_name":"SIMMER: Benchmarking Latent Failures in LLM Executable Planning with a World Model","publication_year":2026,"publication_date":"2026-06-12","ids":{"openalex":"https://openalex.org/W7164837075","doi":"https://doi.org/10.48550/arxiv.2606.14574"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.14574","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.14574","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.14574","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101086021","display_name":"Xiaoxin Lu","orcid":"https://orcid.org/0009-0008-9546-6052"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Xiaoxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077801273","display_name":"Ranran Haoran Zhang","orcid":"https://orcid.org/0000-0002-1005-9757"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Ranran Haoran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138676620","display_name":"Rui Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Rui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.28360000252723694,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.28360000252723694,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.13179999589920044,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.08820000290870667,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.7444999814033508},{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.5616999864578247},{"id":"https://openalex.org/keywords/executable","display_name":"Executable","score":0.5590000152587891},{"id":"https://openalex.org/keywords/futures-studies","display_name":"Futures studies","score":0.4456999897956848},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.41760000586509705},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.41359999775886536},{"id":"https://openalex.org/keywords/plan","display_name":"Plan (archaeology)","score":0.40560001134872437},{"id":"https://openalex.org/keywords/executor","display_name":"Executor","score":0.35760000348091125}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7444999814033508},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7221999764442444},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.5616999864578247},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.5590000152587891},{"id":"https://openalex.org/C64848388","wikidata":"https://www.wikidata.org/wiki/Q188867","display_name":"Futures studies","level":2,"score":0.4456999897956848},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.41760000586509705},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.41359999775886536},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.40560001134872437},{"id":"https://openalex.org/C180591056","wikidata":"https://www.wikidata.org/wiki/Q654437","display_name":"Executor","level":2,"score":0.35760000348091125},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.35589998960494995},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35580000281333923},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.33889999985694885},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33550000190734863},{"id":"https://openalex.org/C46355384","wikidata":"https://www.wikidata.org/wiki/Q726686","display_name":"Compromise","level":2,"score":0.3188999891281128},{"id":"https://openalex.org/C98184364","wikidata":"https://www.wikidata.org/wiki/Q1780131","display_name":"Argument (complex analysis)","level":2,"score":0.3167000114917755},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3158000111579895},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.27079999446868896},{"id":"https://openalex.org/C205372313","wikidata":"https://www.wikidata.org/wiki/Q355645","display_name":"Latent image","level":3,"score":0.2685999870300293},{"id":"https://openalex.org/C112933361","wikidata":"https://www.wikidata.org/wiki/Q2845258","display_name":"Probabilistic latent semantic analysis","level":2,"score":0.2667999863624573},{"id":"https://openalex.org/C192465680","wikidata":"https://www.wikidata.org/wiki/Q1413450","display_name":"Precondition","level":2,"score":0.2630999982357025},{"id":"https://openalex.org/C2776544517","wikidata":"https://www.wikidata.org/wiki/Q189447","display_name":"Unexpected events","level":2,"score":0.2565999925136566},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.25459998846054077}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.14574","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.14574","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.14574","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.14574","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,151],"(LLMs)":[3],"are":[4,111],"increasingly":[5],"deployed":[6],"as":[7],"planners":[8],"for":[9,76,205],"autonomous":[10],"agents":[11],"in":[12,80,90],"household":[13],"environments.":[14],"While":[15],"existing":[16],"benchmarks":[17],"evaluate":[18],"whether":[19],"LLM-generated":[20],"plans":[21,128,163],"execute":[22],"successfully,":[23],"they":[24,63],"overlook":[25],"a":[26,74,84,96,122,202],"critical":[27],"type":[28],"of":[29,162,169],"failure:":[30],"latent":[31,47,78,138,165,188],"failures.":[32,142],"Unlike":[33],"immediate":[34,135],"failures":[35,48,79,189],"that":[36,110,126,148,178],"trigger":[37],"instant":[38],"feedback":[39],"at":[40,153],"execution":[41,54],"time":[42],"and":[43,105,133,140,194],"enable":[44],"timely":[45],"correction,":[46],"do":[49],"not":[50],"immediately":[51],"halt":[52],"plan":[53],"but":[55],"silently":[56],"compromise":[57],"goal":[58],"achievement.":[59],"In":[60],"severe":[61],"cases,":[62],"cause":[64],"irreversible":[65,141,173,195],"harm.":[66],"To":[67],"address":[68],"this":[69],"gap,":[70],"we":[71],"introduce":[72],"SIMMER,":[73],"benchmark":[75],"evaluating":[77],"LLM":[81,208],"planning":[82],"through":[83],"human-curated":[85],"symbolic":[86],"world":[87,97,131],"model":[88,98,132],"grounded":[89],"the":[91,130,167],"kitchen":[92],"domain.":[93],"SIMMER":[94],"defines":[95],"comprising":[99],"77":[100],"actions,":[101],"262":[102],"unique":[103],"objects,":[104],"approximately":[106],"46,800":[107],"possible":[108],"interactions":[109],"semantically":[112],"realistic,":[113],"derived":[114],"from":[115],"real-world":[116],"cooking":[117],"scripts.":[118],"It":[119],"then":[120],"leverages":[121],"state":[123,180],"machine":[124],"executor":[125],"validates":[127],"against":[129],"detects":[134],"precondition":[136],"violations,":[137],"hazards,":[139],"Experiments":[143],"across":[144],"six":[145],"LLMs":[146],"show":[147],"even":[149],"frontier":[150],"achieve":[152],"most":[154],"17%":[155],"error-free":[156],"plans.":[157],"Moreover,":[158],"up":[159,191,198],"to":[160,172,192,199],"56%":[161],"contain":[164],"failures,":[166],"majority":[168],"which":[170],"lead":[171],"consequences.":[174],"We":[175],"further":[176],"demonstrate":[177],"explicit":[179],"reasoning":[181],"via":[182],"counterfactual":[183],"foresight":[184],"simulation":[185],"can":[186],"reduce":[187],"by":[190,197],"72%":[193],"cases":[196],"75%,":[200],"suggesting":[201],"promising":[203],"direction":[204],"more":[206],"robust":[207],"planners.":[209]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-16T00:00:00"}