{"id":"https://openalex.org/W7161680054","doi":"https://doi.org/10.48550/arxiv.2605.17281","title":"ContractBench: Can LLM Agents Preserve Observation Contracts?","display_name":"ContractBench: Can LLM Agents Preserve Observation Contracts?","publication_year":2026,"publication_date":"2026-05-17","ids":{"openalex":"https://openalex.org/W7161680054","doi":"https://doi.org/10.48550/arxiv.2605.17281"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.17281","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.17281","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.17281","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100644806","display_name":"Jicheng Wang","orcid":"https://orcid.org/0000-0003-2744-7059"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jicheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136470526","display_name":"Yifeng He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Yifeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136495220","display_name":"Zili Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zili","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136459265","display_name":"Hanwen Xing","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xing, Hanwen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029857633","display_name":"Arkaprava De","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"De, Arkaprava","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136476212","display_name":"Hao Mark Chen","orcid":"https://orcid.org/0000-0001-9445-035X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.4740999937057495,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.4740999937057495,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.07599999755620956,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.06159999966621399,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6503000259399414},{"id":"https://openalex.org/keywords/byte","display_name":"Byte","score":0.6097000241279602},{"id":"https://openalex.org/keywords/hash-function","display_name":"Hash function","score":0.426800012588501},{"id":"https://openalex.org/keywords/artifact","display_name":"Artifact (error)","score":0.4122999906539917},{"id":"https://openalex.org/keywords/smoothing","display_name":"Smoothing","score":0.40540000796318054},{"id":"https://openalex.org/keywords/session","display_name":"Session (web analytics)","score":0.3596999943256378},{"id":"https://openalex.org/keywords/emulation","display_name":"Emulation","score":0.32839998602867126}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6617000102996826},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6503000259399414},{"id":"https://openalex.org/C43364308","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Byte","level":2,"score":0.6097000241279602},{"id":"https://openalex.org/C99138194","wikidata":"https://www.wikidata.org/wiki/Q183427","display_name":"Hash function","level":2,"score":0.426800012588501},{"id":"https://openalex.org/C2779010991","wikidata":"https://www.wikidata.org/wiki/Q2720909","display_name":"Artifact (error)","level":2,"score":0.4122999906539917},{"id":"https://openalex.org/C3770464","wikidata":"https://www.wikidata.org/wiki/Q775963","display_name":"Smoothing","level":2,"score":0.40540000796318054},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3885999917984009},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.3596999943256378},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3571000099182129},{"id":"https://openalex.org/C149810388","wikidata":"https://www.wikidata.org/wiki/Q5374873","display_name":"Emulation","level":2,"score":0.32839998602867126},{"id":"https://openalex.org/C40305131","wikidata":"https://www.wikidata.org/wiki/Q2616305","display_name":"Obfuscation","level":2,"score":0.3264999985694885},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.3100999891757965},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.301800012588501},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.2922999858856201},{"id":"https://openalex.org/C33762810","wikidata":"https://www.wikidata.org/wiki/Q461671","display_name":"Data integrity","level":2,"score":0.2734000086784363},{"id":"https://openalex.org/C110251889","wikidata":"https://www.wikidata.org/wiki/Q1569697","display_name":"Model checking","level":2,"score":0.25119999051094055}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.17281","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.17281","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.17281","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.17281","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.7042187452316284,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Tool-augmented":[0],"LLM":[1],"agents":[2],"call":[3],"APIs":[4],"whose":[5,22],"intermediate":[6],"outputs,":[7],"such":[8],"as":[9,220],"presigned":[10],"URLs,":[11],"session":[12],"tokens,":[13],"and":[14,44,97,112,120,142,181],"OAuth":[15],"state":[16],"parameters,":[17],"are":[18],"observation":[19,37,167],"contracts:":[20],"artifacts":[21],"later":[23],"use":[24],"is":[25,47,53,110,194],"constrained":[26],"by":[27,56,63],"the":[28,41,105,192,204],"external":[29],"system":[30],"that":[31,36,80,158],"produced":[32],"them.":[33],"We":[34,126,138],"show":[35],"contract":[38],"compliance":[39,211],"(preserving":[40],"temporal":[42],"validity":[43,90],"byte-level":[45],"integrity)":[46],"an":[48,93,101,221],"emergent,":[49],"regression-prone":[50],"capability:":[51],"it":[52],"neither":[54],"guaranteed":[55],"general":[57],"tool-use":[58],"ability":[59],"nor":[60],"consistently":[61],"improved":[62],"larger":[64],"or":[65],"newer":[66],"models.":[67],"To":[68],"measure":[69],"this,":[70],"we":[71],"introduce":[72],"ContractBench,":[73],"a":[74,115,130,170],"benchmark":[75,88],"of":[76],"33":[77],"dual-axis":[78],"tasks":[79],"probe":[81],"two":[82],"orthogonal":[83],"failure":[84,131,217],"modes":[85],"no":[86,147],"existing":[87],"evaluates:":[89],"failures":[91,99],"(using":[92],"artifact":[94],"after":[95],"expiry)":[96],"integrity":[98],"(corrupting":[100],"artifact's":[102],"bytes":[103],"through":[104,212],"observation-to-action":[106],"pipeline).":[107],"Our":[108],"evaluation":[109],"deterministic":[111],"programmatic,":[113],"with":[114,152,166],"virtual":[116],"clock":[117],"controlling":[118],"time":[119],"SHA-256":[121],"hashes":[122],"verifying":[123],"byte":[124],"integrity.":[125],"assign":[127],"each":[128],"outcome":[129],"label":[132],"drawn":[133],"from":[134],"real-world":[135],"API":[136],"specifications.":[137],"evaluate":[139],"38":[140],"models":[141,161],"report":[143],"four":[144],"findings:":[145],"(i)":[146],"evaluated":[148],"model":[149],"clears":[150],"80%,":[151],"Claude-Opus-4.6":[153],"leading":[154],"at":[155,187],"77.8%,":[156],"revealing":[157],"current":[159],"frontier":[160],"still":[162],"fail":[163],"to":[164,185],"comply":[165],"contracts;":[168],"(ii)":[169],"sharp":[171],"within-family":[172],"capability":[173],"cliff":[174,193],"in":[175],"Qwen":[176],"3.5":[177],"between":[178],"4B":[179],"(0%)":[180],"9B":[182],"(56.6%),":[183],"smoothing":[184],"70.7%":[186],"397B-A17B:":[188],"what":[189],"emerges":[190],"across":[191,203],"mid-trajectory":[195],"restraint,":[196],"not":[197],"tool-call":[198],"competence;":[199],"(iii)":[200],"non-monotonic":[201],"scaling":[202],"GPT-5":[205],"family:":[206],"agentic":[207],"post-training":[208],"can":[209],"erode":[210],"sycophancy-driven":[213],"regression;":[214],"(iv)":[215],"our":[216],"taxonomy":[218],"works":[219],"actionable":[222],"in-context":[223],"reward":[224],"signal,":[225],"yielding":[226],"+7.1":[227],"pp":[228],"on":[229],"42":[230],"paired":[231],"GPT-5.1":[232],"failures.":[233]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-20T00:00:00"}