{"id":"https://openalex.org/W7221789366","doi":"https://doi.org/10.48550/arxiv.2610.17724","title":"Evidence-First Agentic Verification","display_name":"Evidence-First Agentic Verification","publication_year":2026,"publication_date":"2026-10-13","ids":{"openalex":"https://openalex.org/W7221789366","doi":"https://doi.org/10.48550/arxiv.2610.17724"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.17724","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.17724","pdf_url":"https://arxiv.org/pdf/2610.17724","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.17724","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119806572","display_name":"Reza Sanayei","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I138006243","display_name":"University of Arizona","ror":"https://ror.org/03m2x1q45","country_code":"US","type":"education","lineage":["https://openalex.org/I138006243"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sanayei, Reza","raw_affiliation_strings":["Scale AI ,","University of Arizona"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Scale AI ,","institution_ids":[]},{"raw_affiliation_string":"University of Arizona","institution_ids":["https://openalex.org/I138006243"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036739449","display_name":"Razvan-Gabriel Dumitru","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dumitru, Razvan-Gabriel","raw_affiliation_strings":["Scale AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Scale AI ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147090657","display_name":"Sunjie Hou","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hou, Sunjie","raw_affiliation_strings":["Scale AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Scale AI ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102422296","display_name":"MohammadHossein Rezaei","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rezaei, MohammadHossein","raw_affiliation_strings":["Scale AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Scale AI ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112264035","display_name":"Haris Riaz","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I138006243","display_name":"University of Arizona","ror":"https://ror.org/03m2x1q45","country_code":"US","type":"education","lineage":["https://openalex.org/I138006243"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Riaz, Haris","raw_affiliation_strings":["Scale AI ,","University of Arizona"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Scale AI ,","institution_ids":[]},{"raw_affiliation_string":"University of Arizona","institution_ids":["https://openalex.org/I138006243"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084170301","display_name":"Robert Vacareanu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vacareanu, Robert","raw_affiliation_strings":["Scale AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Scale AI ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5158751345","display_name":"Minglai Yang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Minglai","raw_affiliation_strings":["Scale AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Scale AI ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5158684734","display_name":"Ying Liu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Ying","raw_affiliation_strings":["Scale AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Scale AI ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047699502","display_name":"Mihai Surdeanu","orcid":"https://orcid.org/0000-0001-6956-8030","observed_orcids":["https://orcid.org/0000-0001-6956-8030"]},"institutions":[{"id":"https://openalex.org/I138006243","display_name":"University of Arizona","ror":"https://ror.org/03m2x1q45","country_code":"US","type":"education","lineage":["https://openalex.org/I138006243"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Surdeanu, Mihai","raw_affiliation_strings":["University of Arizona"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Arizona","institution_ids":["https://openalex.org/I138006243"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5158728035","display_name":"Daniel Yue Zhang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Daniel Yue","raw_affiliation_strings":["Scale AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Scale AI ,","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.7031000256538391,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.7031000256538391,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.04610000178217888,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10142","display_name":"Formal Methods in Verification","score":0.014399999752640724,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/agent-verification","display_name":"agent verification","score":0.41999998688697815},{"id":"https://openalex.org/keywords/execution-trace-analysis","display_name":"execution trace analysis","score":0.44999998807907104},{"id":"https://openalex.org/keywords/personal-assistant-agents","display_name":"personal assistant agents","score":0.546999990940094},{"id":"https://openalex.org/keywords/long-horizon-tasks","display_name":"long-horizon tasks","score":0.5759999752044678},{"id":"https://openalex.org/keywords/rubric-based-assessment","display_name":"rubric-based assessment","score":0.652999997138977},{"id":"https://openalex.org/keywords/fault-injection","display_name":"fault injection","score":0.5429999828338623},{"id":"https://openalex.org/keywords/sauron","display_name":"Sauron","score":0.503000020980835}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6966000199317932},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6543999910354614},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.5145999789237976},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4729999899864197},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.4374000132083893},{"id":"https://openalex.org/C2777548347","wikidata":"https://www.wikidata.org/wiki/Q5456937","display_name":"Flagging","level":2,"score":0.4372999966144562},{"id":"https://openalex.org/C2777338717","wikidata":"https://www.wikidata.org/wiki/Q1762621","display_name":"Vendor","level":2,"score":0.390500009059906},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3727000057697296},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.36239999532699585},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.35600000619888306},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.34850001335144043},{"id":"https://openalex.org/C63584917","wikidata":"https://www.wikidata.org/wiki/Q333286","display_name":"Bounding overwatch","level":2,"score":0.33000001311302185},{"id":"https://openalex.org/C2779338814","wikidata":"https://www.wikidata.org/wiki/Q5179285","display_name":"Covert","level":2,"score":0.30480000376701355},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.2904999852180481},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2800999879837036},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2791999876499176},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.27469998598098755},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.26570001244544983},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.2653000056743622},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.26159998774528503},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.25600001215934753},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2506999969482422}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.17724","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.17724","pdf_url":"https://arxiv.org/pdf/2610.17724","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.17724","doi":"https://doi.org/10.48550/arxiv.2610.17724","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.17724","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.17724","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.17724","pdf_url":"https://arxiv.org/pdf/2610.17724","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7221789366.pdf","grobid_xml":"https://content.openalex.org/works/W7221789366.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W4310290453","https://openalex.org/W4389519598","https://openalex.org/W4391876619","https://openalex.org/W4399795554","https://openalex.org/W4402671800","https://openalex.org/W4415235458","https://openalex.org/W4416034201","https://openalex.org/W4416037109","https://openalex.org/W6891853064","https://openalex.org/W6948017749","https://openalex.org/W7119980254","https://openalex.org/W7125134911","https://openalex.org/W7133230429","https://openalex.org/W7154374450","https://openalex.org/W7155622066","https://openalex.org/W7160904950","https://openalex.org/W7161016054","https://openalex.org/W7172276332","https://openalex.org/W7201873470"],"related_works":["https://openalex.org/W4417025121","https://openalex.org/W7216142797","https://openalex.org/W7165108501","https://openalex.org/W7222948915","https://openalex.org/W7166802094","https://openalex.org/W7203446082","https://openalex.org/W7165177831","https://openalex.org/W7216474086","https://openalex.org/W7155245052","https://openalex.org/W7117842883"],"abstract_inverted_index":{"Long-horizon,":[0],"semi-verifiable":[1],"agentic":[2,63],"tasks":[3,16],"often":[4],"rely":[5],"on":[6,73,125,156,215],"rubric-based":[7],"evaluation,":[8],"but":[9],"the":[10,21,25,119,135,193],"evidence":[11,33],"needed":[12],"to":[13],"assess":[14],"these":[15],"can":[17,54],"be":[18],"spread":[19],"across":[20],"environment":[22],"state":[23,68],"and":[24,69,71,86,95,98,202,211],"execution":[26],"trace.":[27,195],"A":[28],"judge":[29,214],"may":[30],"overlook":[31],"this":[32],"or":[34],"accept":[35],"unsupported":[36],"claims,":[37],"including":[38],"an":[39,61,102],"agent's":[40],"own":[41,213],"reports":[42],"of":[43,89,105,122,168,176,183],"success.":[44],"When":[45],"rubric":[46],"verdicts":[47],"serve":[48],"as":[49],"reinforcement-learning":[50],"rewards,":[51],"such":[52],"errors":[53],"reinforce":[55],"undesirable":[56],"behavior.":[57],"We":[58],"introduce":[59],"Sauron,":[60],"evidence-first":[62],"verifier":[64],"that":[65],"investigates":[66],"both":[67,209],"trace":[70],"relies":[72],"a":[74,153,160,198],"claim":[75],"only":[76,181],"after":[77],"independently":[78],"validating":[79],"it":[80,141],"against":[81],"its":[82,106],"source.":[83],"Bounded":[84],"queries":[85],"joint":[87],"investigation":[88],"related":[90],"criteria":[91,112],"reduce":[92],"context":[93],"use":[94],"redundant":[96],"work,":[97],"each":[99],"verdict":[100],"carries":[101],"auditable":[103],"receipt":[104],"supporting":[107],"evidence.":[108],"Across":[109],"2,586":[110],"human-labeled":[111],"from":[113],"100":[114],"personal-assistant":[115],"episodes,":[116],"Sauron":[117,173,207],"achieves":[118],"highest":[120],"F1":[121,144],"five":[123],"verifiers":[124],"all":[126,216],"eight":[127],"backbones":[128],"tested":[129],"while":[130,179],"costing":[131],"less":[132],"than":[133],"Gandalf,":[134],"strongest":[136],"baseline.":[137],"With":[138],"GPT-5.6":[139],"Sol,":[140],"reaches":[142],"0.96":[143],"versus":[145],"Gandalf's":[146],"0.92":[147],"at":[148],"23%":[149],"lower":[150],"cost,":[151],"with":[152],"wider":[154],"margin":[155],"process":[157],"criteria.":[158],"On":[159,196],"paired":[161],"fault-injection":[162],"suite":[163],"covering":[164],"13":[165],"families,":[166],"several":[167],"which":[169],"simulate":[170],"reward":[171],"hacking,":[172],"detects":[174],"97%":[175],"injected":[177],"failures":[178],"flagging":[180],"2%":[182],"clean":[184],"controls,":[185],"outperforming":[186],"baselines":[187],"even":[188],"when":[189],"they":[190],"are":[191],"given":[192],"full":[194],"TRAIL,":[197],"benchmark":[199],"for":[200],"detecting":[201],"localizing":[203],"real":[204],"agent":[205],"errors,":[206],"leads":[208],"Gandalf":[210],"TRAIL's":[212],"three":[217],"quality":[218],"metrics.":[219]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-15T00:00:00"}