{"id":"https://openalex.org/W7172031506","doi":"https://doi.org/10.48550/arxiv.2607.27518","title":"Automated Transcript Analysis for Detecting Flaws in Agentic Benchmarks","display_name":"Automated Transcript Analysis for Detecting Flaws in Agentic Benchmarks","publication_year":2026,"publication_date":"2026-07-29","ids":{"openalex":"https://openalex.org/W7172031506","doi":"https://doi.org/10.48550/arxiv.2607.27518"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.27518","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.27518","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.27518","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5144145314","display_name":"Jeff Mohl","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mohl, Jeff","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126842451","display_name":"Nelson Gardner-Challis","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gardner-Challis, Nelson","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055765010","display_name":"Magda Dubois","orcid":"https://orcid.org/0000-0002-5396-1855"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dubois, Magda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087769785","display_name":"Harry Coppock","orcid":"https://orcid.org/0000-0003-2404-8319"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Coppock, Harry","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5144131663","display_name":"Benjamin Allan-Rahill","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Allan-Rahill, Benjamin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5144176507","display_name":"Kaelan Yim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yim, Kaelan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044968824","display_name":"Damian S\u00f3jka","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"S\u00f3jka, Damian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5144132563","display_name":"James Mann","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mann, James","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5074071057","display_name":"Justin Olive","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Olive, Justin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.2184000015258789,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.2184000015258789,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11492","display_name":"Academic integrity and plagiarism","score":0.09969999641180038,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.09189999848604202,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/quality-assurance","display_name":"Quality assurance","score":0.5461999773979187},{"id":"https://openalex.org/keywords/standardization","display_name":"Standardization","score":0.5167999863624573},{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.4668000042438507},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.46299999952316284},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.4474000036716461},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.41990000009536743},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4163999855518341},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.3853999972343445}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.682699978351593},{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.5461999773979187},{"id":"https://openalex.org/C188087704","wikidata":"https://www.wikidata.org/wiki/Q369577","display_name":"Standardization","level":2,"score":0.5167999863624573},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.4668000042438507},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.46299999952316284},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4474000036716461},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44119998812675476},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.41990000009536743},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4163999855518341},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3853999972343445},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.37610000371932983},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.351500004529953},{"id":"https://openalex.org/C2779751349","wikidata":"https://www.wikidata.org/wiki/Q1474480","display_name":"Scanner","level":2,"score":0.34619998931884766},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.3345000147819519},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.3330000042915344},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.33230000734329224},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.3319000005722046},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32829999923706055},{"id":"https://openalex.org/C64869954","wikidata":"https://www.wikidata.org/wiki/Q1859747","display_name":"False positive paradox","level":2,"score":0.3156999945640564},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.30959999561309814},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2919999957084656},{"id":"https://openalex.org/C92446256","wikidata":"https://www.wikidata.org/wiki/Q3306762","display_name":"Data validation","level":2,"score":0.27000001072883606}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.27518","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.27518","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.27518","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.27518","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.7170718908309937,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Capabilities":[0],"of":[1,31,76,112,188],"frontier":[2],"models":[3],"are":[4],"often":[5],"assessed":[6],"using":[7,191],"agentic":[8],"benchmarks.":[9,115],"To":[10],"trust":[11],"these":[12,182],"results,":[13],"benchmarks":[14,32],"must":[15],"accurately":[16],"measure":[17],"what":[18],"they":[19],"claim":[20],"to":[21,48,72,96,131,158,161,195],"and":[22,50,86,100,143,151],"be":[23,132,159],"free":[24],"from":[25],"invalidating":[26],"flaws.":[27],"Previous":[28],"manual":[29,44,136],"audits":[30],"such":[33],"as":[34,185],"SWE-Bench-Verified":[35],"have":[36],"uncovered":[37],"several":[38,119,155],"validity":[39,77],"issues":[40,122],"in":[41,123,173],"transcripts.":[42],"However,":[43],"review":[45],"is":[46,52],"difficult":[47],"scale,":[49],"it":[51],"unclear":[53],"whether":[54],"automated":[55,192],"methods":[56],"can":[57],"reliably":[58],"surface":[59],"flaws":[60],"that":[61,177],"compromise":[62],"benchmark":[63,197],"validity.":[64],"In":[65],"this":[66],"paper,":[67],"we":[68],"developed":[69],"AI":[70],"scanners":[71,103,117,163],"detect":[73],"four":[74],"types":[75],"issues:":[78],"ground":[79],"truth":[80],"access,":[81],"tool":[82],"failure,":[83],"guessing":[84],"vulnerability,":[85],"answer":[87],"format":[88],"ambiguity.":[89],"We":[90,153],"produced":[91],"grading":[92],"rubrics":[93],"for":[94,164,190],"each":[95],"instruct":[97],"human":[98,105],"labeling,":[99],"evaluated":[101],"the":[102,174],"against":[104],"labels":[106],"on":[107],"a":[108,186],"held-out":[109],"test":[110],"set":[111],"Inspect":[113],"Evals":[114],"Our":[116],"identified":[118],"verified":[120],"quality":[121,166,198],"five":[124],"widely":[125],"used":[126],"benchmarks,":[127,149],"including":[128,169],"cases":[129,140],"unlikely":[130],"caught":[133],"by":[134],"random":[135],"inspection.":[137],"Not":[138],"all":[139],"were":[141],"identified,":[142],"scanner":[144,179],"performance":[145],"varied":[146],"substantially":[147],"across":[148],"criteria":[150],"models.":[152],"highlight":[154],"open":[156],"challenges":[157],"addressed":[160],"improve":[162],"stronger":[165],"assurance":[167],"claims,":[168],"broader":[170],"standardization":[171],"gaps":[172],"evaluation":[175],"field":[176],"degrade":[178],"performance.":[180],"Together,":[181],"results":[183],"serve":[184],"proof":[187],"concept":[189],"transcript":[193],"analysis":[194],"audit":[196],"more":[199],"broadly.":[200]},"counts_by_year":[],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2026-08-01T00:00:00"}