{"id":"https://openalex.org/W7163875934","doi":"https://doi.org/10.48550/arxiv.2606.07379","title":"Do Coding Agents Deceive Us? Detecting and Preventing Cheating via Capped Evaluation with Randomized Tests","display_name":"Do Coding Agents Deceive Us? Detecting and Preventing Cheating via Capped Evaluation with Randomized Tests","publication_year":2026,"publication_date":"2026-06-05","ids":{"openalex":"https://openalex.org/W7163875934","doi":"https://doi.org/10.48550/arxiv.2606.07379"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.07379","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07379","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.07379","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138152348","display_name":"Thanawat Lodkaew","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lodkaew, Thanawat","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070927069","display_name":"Johannes Ackermann","orcid":"https://orcid.org/0000-0001-7141-8912"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ackermann, Johannes","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091587258","display_name":"Soichiro Nishimori","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nishimori, Soichiro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083583938","display_name":"Nontawat Charoenphakdee","orcid":"https://orcid.org/0000-0002-0214-4943"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Charoenphakdee, Nontawat","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138117608","display_name":"Masashi Sugiyama","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sugiyama, Masashi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138151031","display_name":"Takashi Ishida","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ishida, Takashi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.3961000144481659,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.3961000144481659,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.18729999661445618,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.1664000004529953,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cheating","display_name":"Cheating","score":0.9591000080108643},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.6219000220298767},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.5651999711990356},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5157999992370605},{"id":"https://openalex.org/keywords/randomized-experiment","display_name":"Randomized experiment","score":0.43650001287460327},{"id":"https://openalex.org/keywords/deception","display_name":"Deception","score":0.35089999437332153}],"concepts":[{"id":"https://openalex.org/C2778024590","wikidata":"https://www.wikidata.org/wiki/Q2357432","display_name":"Cheating","level":2,"score":0.9591000080108643},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7032999992370605},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.6219000220298767},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6017000079154968},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5658000111579895},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.5651999711990356},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5157999992370605},{"id":"https://openalex.org/C155108698","wikidata":"https://www.wikidata.org/wiki/Q1231081","display_name":"Randomized experiment","level":2,"score":0.43650001287460327},{"id":"https://openalex.org/C2779267917","wikidata":"https://www.wikidata.org/wiki/Q170028","display_name":"Deception","level":2,"score":0.35089999437332153},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3402999937534332},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.30970001220703125},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.2793000042438507},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.2782999873161316},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.26460000872612},{"id":"https://openalex.org/C143299363","wikidata":"https://www.wikidata.org/wiki/Q900584","display_name":"Attribution","level":2,"score":0.2605000138282776},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.26019999384880066}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.07379","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07379","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.07379","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07379","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"A":[0],"growing":[1],"failure":[2],"mode":[3],"in":[4],"agent":[5],"evaluation":[6,15,31,66],"and":[7,78,119],"training":[8],"is":[9,57],"that":[10,109,126],"models":[11,125],"can":[12],"achieve":[13],"high":[14],"scores":[16,32,67,71],"by":[17],"exploiting":[18],"shortcuts":[19],"instead":[20],"of":[21,36,82,117],"solving":[22],"the":[23,74,95,102,129],"intended":[24,130],"task,":[25],"producing":[26],"deceptive":[27],"performance.":[28],"This":[29,62],"makes":[30],"unreliable":[33],"as":[34],"measures":[35],"true":[37],"task-solving":[38],"ability.":[39],"We":[40],"propose":[41,88],"CapCode,":[42],"a":[43,68,90],"framework":[44],"for":[45],"constructing":[46],"coding":[47],"datasets":[48,107],"with":[49],"randomized":[50],"tests":[51],"whose":[52],"best":[53],"achievable":[54],"non-cheating":[55],"performance":[56,115],"deliberately":[58],"capped":[59],"below":[60],"one.":[61],"capped-performance":[63],"design":[64,92],"gives":[65],"clearer":[69],"interpretation:":[70],"substantially":[72],"above":[73],"cap":[75],"are":[76],"implausible":[77],"therefore":[79],"provide":[80],"evidence":[81],"cheating.":[83],"To":[84],"prevent":[85],"cheating,":[86],"we":[87],"CapReward,":[89],"reward":[91],"based":[93],"on":[94],"CapCode":[96,110],"principle":[97],"to":[98],"discourage":[99],"optimization":[100],"beyond":[101],"cap.":[103],"Experiments":[104],"across":[105],"multiple":[106],"show":[108],"detects":[111],"cheating":[112,122],"while":[113],"preserving":[114],"ranking":[116],"models,":[118],"CapReward":[120],"reduces":[121],"behavior,":[123],"yielding":[124],"better":[127],"follow":[128],"task":[131],"specification.":[132]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-09T00:00:00"}