{"id":"https://openalex.org/W7224414849","doi":"https://doi.org/10.5281/zenodo.23752398","title":"Benchmark Honesty: Recovery and Abstention after Tool Failure","display_name":"Benchmark Honesty: Recovery and Abstention after Tool Failure","publication_year":2026,"publication_date":"2026-10-29","ids":{"openalex":"https://openalex.org/W7224414849","doi":"https://doi.org/10.5281/zenodo.23752398"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.23752398","doi":"https://doi.org/10.5281/zenodo.23752398","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23752398","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.23752398","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5158865149","display_name":"Pooria Arab","orcid":"https://orcid.org/0009-0005-9964-6841","observed_orcids":["https://orcid.org/0009-0005-9964-6841"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Pooria Arab","raw_affiliation_strings":["Independent researcher"],"raw_orcid":"https://orcid.org/0009-0005-9964-6841","affiliations":[{"raw_affiliation_string":"Independent researcher","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5158865149"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.36910000443458557,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.36910000443458557,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.0989999994635582,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11492","display_name":"Academic integrity and plagiarism","score":0.08760000020265579,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/tool-failure","display_name":"tool failure","score":0.675000011920929},{"id":"https://openalex.org/keywords/fault-recovery","display_name":"fault recovery","score":0.5329999923706055},{"id":"https://openalex.org/keywords/abstention","display_name":"abstention","score":0.7990000247955322},{"id":"https://openalex.org/keywords/confidence-calibration","display_name":"confidence calibration","score":0.4350000023841858}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8453999757766724},{"id":"https://openalex.org/C2777293324","wikidata":"https://www.wikidata.org/wiki/Q337349","display_name":"Honesty","level":2,"score":0.7639999985694885},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6115000247955322},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.5564000010490417},{"id":"https://openalex.org/C2778689934","wikidata":"https://www.wikidata.org/wiki/Q1313396","display_name":"Headline","level":2,"score":0.5141000151634216},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.44699999690055847},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4108999967575073},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37720000743865967},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3127000033855438},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.30480000376701355},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.29660001397132874},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.2870999872684479},{"id":"https://openalex.org/C2778067643","wikidata":"https://www.wikidata.org/wiki/Q166507","display_name":"Interval (graph theory)","level":2,"score":0.2851000130176544},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C163164238","wikidata":"https://www.wikidata.org/wiki/Q2737027","display_name":"Failure rate","level":2,"score":0.27480000257492065},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.27239999175071716},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.2694999873638153}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.23752398","doi":"https://doi.org/10.5281/zenodo.23752398","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23752398","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23752399","doi":"https://doi.org/10.5281/zenodo.23752399","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23752399","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.23752398","doi":"https://doi.org/10.5281/zenodo.23752398","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23752398","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7202192897","https://openalex.org/W7169550126","https://openalex.org/W7214566192","https://openalex.org/W7056237847","https://openalex.org/W7204729020","https://openalex.org/W1973208754","https://openalex.org/W7196923922","https://openalex.org/W4389987507","https://openalex.org/W7110661800","https://openalex.org/W7212068589"],"abstract_inverted_index":{"Honesty":[0,10],"after":[1],"tool":[2],"failure":[3],"requires":[4],"both":[5],"recovery":[6,52],"and":[7,22,40,48,67],"restraint.":[8],"Benchmark":[9],"tests":[11],"this":[12],"distinction":[13],"with":[14,32],"60":[15],"fictional":[16],"lookup":[17],"episodes:":[18],"30":[19,23],"recoverable":[20],"failures":[21],"unavailable":[24],"records.":[25],"Its":[26],"headline":[27],"score":[28],"combines":[29],"evidence-backed":[30,33],"answering":[31],"abstention.":[34],"We":[35],"evaluate":[36],"10":[37],"scripted":[38],"controls":[39,69],"a":[41],"confidence":[42,90],"sweep":[43],"through":[44],"the":[45],"benchmark":[46],"environment":[47],"grader.":[49],"The":[50],"evidence-reading":[51],"script":[53],"achieves":[54],"balanced":[55,71,92],"success":[56,72,93],"1.000":[57,63],"[1.000,":[58],"1.000];":[59],"each":[60],"component":[61,83],"is":[62,125],"[0.886,":[64],"1.000].":[65],"Always-abstain":[66],"no-retry":[68],"achieve":[70],"0.000":[73],"[0.000,":[74],"0.000].":[75],"Balanced-success":[76],"intervals":[77],"use":[78,85],"95%":[79,86],"item":[80],"bootstrap":[81],"bounds;":[82],"rates":[84],"Wilson":[87],"bounds.":[88],"Changing":[89],"leaves":[91],"unchanged":[94],"while":[95],"changing":[96],"Brier":[97],"loss.":[98],"All":[99],"14":[100],"repeated":[101],"conditions":[102],"agree":[103],"in":[104,117],"their":[105],"semantic":[106],"traces.":[107],"These":[108],"results":[109],"validate":[110],"selected":[111],"scoring":[112],"distinctions,":[113],"not":[114],"model":[115,121],"honesty":[116],"deployment.":[118],"No":[119],"learned":[120],"or":[122],"human":[123],"participant":[124],"evaluated.":[126]},"counts_by_year":[],"updated_date":"2026-10-31T12:09:11.096373","created_date":"2026-10-30T00:00:00"}