{"id":"https://openalex.org/W7167427324","doi":"https://doi.org/10.1145/3805760.3814919","title":"How Robustly Do LLMs Understand Execution Semantics?","display_name":"How Robustly Do LLMs Understand Execution Semantics?","publication_year":2026,"publication_date":"2026-07-05","ids":{"openalex":"https://openalex.org/W7167427324","doi":"https://doi.org/10.1145/3805760.3814919"},"language":null,"primary_location":{"id":"doi:10.1145/3805760.3814919","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805760.3814919","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd ACM International Conference on AI-Powered Software","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805760.3814919","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008515825","display_name":"C. Katharina Spie\u00df","orcid":"https://orcid.org/0009-0000-5932-7022"},"institutions":[{"id":"https://openalex.org/I84218800","display_name":"University of California, Davis","ror":"https://ror.org/05rrcem69","country_code":"US","type":"education","lineage":["https://openalex.org/I84218800"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Claudio Spiess","raw_affiliation_strings":["University of California at Davis, Davis, USA"],"raw_orcid":"https://orcid.org/0009-0000-5932-7022","affiliations":[{"raw_affiliation_string":"University of California at Davis, Davis, USA","institution_ids":["https://openalex.org/I84218800"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036744986","display_name":"Pr\u00e9mkumar D\u00e9vanbu","orcid":"https://orcid.org/0000-0002-4346-5276"},"institutions":[{"id":"https://openalex.org/I84218800","display_name":"University of California, Davis","ror":"https://ror.org/05rrcem69","country_code":"US","type":"education","lineage":["https://openalex.org/I84218800"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Prem Devanbu","raw_affiliation_strings":["University of California at Davis, Davis, USA"],"raw_orcid":"https://orcid.org/0000-0002-4346-5276","affiliations":[{"raw_affiliation_string":"University of California at Davis, Davis, USA","institution_ids":["https://openalex.org/I84218800"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076587279","display_name":"Earl T. Barr","orcid":"https://orcid.org/0000-0003-0771-7891"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Earl T. Barr","raw_affiliation_strings":["University College London, London, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0003-0771-7891","affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.8946534,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"288","last_page":"298"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.6248000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.6248000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10126","display_name":"Logic, programming, and type systems","score":0.0828000009059906,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.03920000046491623,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.5763999819755554},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.44119998812675476},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.414000004529953},{"id":"https://openalex.org/keywords/through-the-lens-metering","display_name":"Through-the-lens metering","score":0.3598000109195709},{"id":"https://openalex.org/keywords/frontier","display_name":"Frontier","score":0.3357999920845032},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.3328999876976013}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5763999819755554},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5169000029563904},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.44119998812675476},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.414000004529953},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.38339999318122864},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3643999993801117},{"id":"https://openalex.org/C43091099","wikidata":"https://www.wikidata.org/wiki/Q1067788","display_name":"Through-the-lens metering","level":3,"score":0.3598000109195709},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.3357999920845032},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.3328999876976013},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.33070001006126404},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.32170000672340393},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.3131999969482422},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2944999933242798},{"id":"https://openalex.org/C207390915","wikidata":"https://www.wikidata.org/wiki/Q1230525","display_name":"Divergence (linguistics)","level":2,"score":0.28839999437332153},{"id":"https://openalex.org/C2779714256","wikidata":"https://www.wikidata.org/wiki/Q25305062","display_name":"Multiple Models","level":2,"score":0.2808000147342682},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.27880001068115234}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805760.3814919","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805760.3814919","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd ACM International Conference on AI-Powered Software","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805760.3814919","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805760.3814919","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd ACM International Conference on AI-Powered Software","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.4348847270011902,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W1978304215","https://openalex.org/W2148437670","https://openalex.org/W2741561716","https://openalex.org/W3004658838","https://openalex.org/W4308731473","https://openalex.org/W4376606665","https://openalex.org/W4392425548","https://openalex.org/W4400582761","https://openalex.org/W4401306886","https://openalex.org/W4403536473","https://openalex.org/W4403746689","https://openalex.org/W4405655184","https://openalex.org/W4407989189","https://openalex.org/W4411359806","https://openalex.org/W4411552659","https://openalex.org/W4412887691","https://openalex.org/W4412944815","https://openalex.org/W4414155484","https://openalex.org/W4414281281","https://openalex.org/W4414790686","https://openalex.org/W4414820678","https://openalex.org/W4416167123","https://openalex.org/W4416519743","https://openalex.org/W6929367728","https://openalex.org/W6967058914","https://openalex.org/W7106506476","https://openalex.org/W7125135101"],"related_works":[],"abstract_inverted_index":{"LLMs":[0,22],"demonstrate":[1],"remarkable":[2],"reasoning":[3,49],"capabilities,":[4],"yet":[5],"whether":[6],"they":[7],"utilize":[8],"internal":[9],"world":[10],"models":[11,50,104,161],"or":[12],"rely":[13],"on":[14,83,111,122,144],"sophisticated":[15],"pattern":[16],"matching":[17],"remains":[18],"open.":[19],"We":[20,127],"study":[21,128],"through":[23],"the":[24,68,84,123,139,145,158,168],"lens":[25],"of":[26,28,81,125,141,170],"robustness":[27],"their":[29],"code":[30,63,175],"understanding":[31],"using":[32,171],"a":[33,41,78],"standard":[34],"program-output":[35],"prediction":[36,119],"task.":[37],"Our":[38,151],"results":[39],"reveal":[40],"stark":[42],"divergence":[43],"in":[44,134,157],"model":[45,70],"behavior:":[46],"while":[47],"open-source":[48],"(DeepSeek-R1":[51],"family)":[52],"maintain":[53],"stable,":[54],"albeit":[55],"somewhat":[56],"lower":[57],"accuracies":[58],"(38%":[59],"to":[60,130,147,155,173],"67%)":[61],"under":[62],"transformations":[64],"&":[65],"input":[66],"perturbations,":[67],"frontier":[69],"GPT-5.2":[71],"exhibits":[72],"surprising":[73],"instruction-sensitive":[74],"brittleness.":[75],"Despite":[76],"achieving":[77],"near-perfect":[79],"score":[80],"99%":[82],"original,":[85],"unperturbed":[86],"CRUXEval":[87],"benchmark,":[88],"perturbed":[89,112],"inputs":[90,113],"trigger":[91],"accuracy":[92],"declines":[93],"between":[94],"20%":[95],"and":[96,117,137,164,166],"24%.":[97],"In":[98],"addition,":[99],"we":[100],"find":[101],"that":[102,114,118],"many":[103],"perform":[105],"much":[106],"worse":[107],"at":[108],"predicting":[109],"behavior":[110],"raise":[115],"exceptions,":[116],"performance":[120],"depends":[121],"kind":[124],"exception.":[126],"remedies":[129,143],"address":[131],"this":[132],"deficiency":[133],"exception":[135],"prediction,":[136],"evaluate":[138,174],"effect":[140],"these":[142],"ability":[146],"predict":[148],"non-exception":[149],"behaviors.":[150],"findings":[152],"both":[153],"point":[154],"limitations":[156],"way":[159],"all":[160],"understand":[162],"code,":[163],"reinforce":[165],"extend":[167],"value":[169],"perturbation":[172],"models.":[176]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-07-06T00:00:00"}