{"id":"https://openalex.org/W7220958569","doi":"https://doi.org/10.48550/arxiv.2610.09541","title":"Certified by Abstention: Distribution-Free Guarantees for Chain-of-Thought Verifiers at Small Calibration Budgets","display_name":"Certified by Abstention: Distribution-Free Guarantees for Chain-of-Thought Verifiers at Small Calibration Budgets","publication_year":2026,"publication_date":"2026-10-07","ids":{"openalex":"https://openalex.org/W7220958569","doi":"https://doi.org/10.48550/arxiv.2610.09541"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.09541","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.09541","pdf_url":"https://arxiv.org/pdf/2610.09541","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.09541","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5151645979","display_name":"Arjun Balaji","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Balaji, Arjun","raw_affiliation_strings":["Columbia University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Columbia University","institution_ids":["https://openalex.org/I78577930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5151645979"],"corresponding_institution_ids":["https://openalex.org/I78577930"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.09610000252723694,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.09610000252723694,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.0778999999165535,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.06030000001192093,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/selective-inference","display_name":"selective inference","score":0.4650000035762787},{"id":"https://openalex.org/keywords/abstention","display_name":"abstention","score":0.5509999990463257},{"id":"https://openalex.org/keywords/certificate-validity","display_name":"certificate validity","score":0.5450000166893005},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.46700000762939453},{"id":"https://openalex.org/keywords/failure-probability-bounds","display_name":"failure probability bounds","score":0.335999995470047}],"concepts":[{"id":"https://openalex.org/C96865113","wikidata":"https://www.wikidata.org/wiki/Q2946816","display_name":"Certificate","level":2,"score":0.8396000266075134},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.5016999840736389},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4927999973297119},{"id":"https://openalex.org/C46304622","wikidata":"https://www.wikidata.org/wiki/Q374814","display_name":"Certification","level":2,"score":0.4722999930381775},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4325999915599823},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.3801000118255615},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3603000044822693},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.31520000100135803},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.31040000915527344},{"id":"https://openalex.org/C162307627","wikidata":"https://www.wikidata.org/wiki/Q204833","display_name":"Enhanced Data Rates for GSM Evolution","level":2,"score":0.29649999737739563},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2833999991416931},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2777999937534332},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2655999958515167},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.262800008058548}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.09541","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.09541","pdf_url":"https://arxiv.org/pdf/2610.09541","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.09541","doi":"https://doi.org/10.48550/arxiv.2610.09541","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.09541","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.09541","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.09541","pdf_url":"https://arxiv.org/pdf/2610.09541","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220958569.pdf","grobid_xml":"https://content.openalex.org/works/W7220958569.grobid-xml"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W2794325560","https://openalex.org/W3210904070","https://openalex.org/W4285429195","https://openalex.org/W4327526719","https://openalex.org/W4405655184","https://openalex.org/W4414939355","https://openalex.org/W4414946584","https://openalex.org/W4417465430","https://openalex.org/W6948017749"],"related_works":["https://openalex.org/W7220747484","https://openalex.org/W7214084344","https://openalex.org/W7171491351","https://openalex.org/W7222914768","https://openalex.org/W7161268714","https://openalex.org/W7212037434","https://openalex.org/W7162403334","https://openalex.org/W7161012223","https://openalex.org/W7222991846","https://openalex.org/W4402671733"],"abstract_inverted_index":{"Signals":[0],"that":[1,66,89,203],"predict":[2],"whether":[3],"a":[4,18,21,40,68,87,103,134,160,182,195,234],"chain-of-thought":[5],"(CoT)":[6],"trace":[7],"is":[8,59,100],"correct":[9],"are":[10],"compared":[11],"by":[12,61,83],"AUC,":[13],"but":[14,119],"deploying":[15],"one":[16],"requires":[17],"threshold":[19],"with":[20,70,105],"guarantee.":[22],"We":[23,192],"ask":[24],"what":[25,238],"distribution-free":[26],"selective":[27],"guarantees":[28],"deliver":[29],"for":[30,137],"CoT":[31],"verifiers":[32],"at":[33,113,145,217],"realistic":[34],"calibration":[35,117],"budgets":[36],"of":[37,78,116,124,176],"tens":[38],"to":[39,122,171,224],"few":[41],"hundred":[42],"labelled":[43],"problems,":[44],"using":[45],"seven":[46],"open":[47],"models,":[48],"five":[49],"verifier":[50,262],"signals":[51],"and":[52,95,133,148,164,214,256],"37,000":[53],"graded":[54],"traces.":[55],"The":[56],"central":[57],"observation":[58],"validity":[60],"abstention:":[62],"an":[63,79,165,180],"$(\u03b1,\u03b4)$-valid":[64],"procedure":[65],"issues":[67],"certificate":[69,81,88,110,156,209,235],"probability":[71,77],"$P_{\\rm":[72],"fire}$":[73],"bounds":[74],"the":[75,108,149,154,174,177,189,207,218,227,245,251,261,266,269,279],"failure":[76,271],"issued":[80],"only":[82],"$\u03b4/P_{\\rm":[84],"fire}$,":[85],"so":[86],"rarely":[90],"fires":[91],"can":[92],"be":[93],"valid":[94,199],"wrong":[96],"every":[97,211],"time":[98],"it":[99,128,263],"used.":[101],"In":[102],"simulation":[104],"known":[106],"risk":[107],"standard":[109,155],"fails":[111],"in":[112,120,126],"most":[114],"0.3%":[115],"draws":[118],"up":[121],"69%":[123],"those":[125],"which":[127],"fires.":[129],"A":[130],"certification":[131],"floor":[132,228],"lattice":[135],"condition":[136],"Benjamini-Hochberg":[138],"conformal":[139],"selection":[140,259],"explain":[141],"why":[142],"certificates":[143],"abstain":[144],"these":[146],"budgets,":[147],"data":[150],"bear":[151],"them":[152],"out:":[153],"returns":[157],"nothing":[158],"or":[159],"large":[161],"accepted":[162,248],"set,":[163],"unreadable":[166],"residual-stream":[167],"probe":[168],"buys":[169],"two":[170],"three":[172],"times":[173],"coverage":[175,216,231],"readable":[178,190],"signals,":[179],"edge":[181],"cross-fitted":[183],"reconstruction":[184],"cannot":[185,236],"recover":[186],"linearly":[187],"from":[188,222],"features.":[191],"then":[193],"give":[194],"floor-started":[196],"fixed-sequence":[197],"certificate,":[198],"without":[200],"monotonicity":[201],"assumptions,":[202],"covers":[204],"more":[205],"than":[206],"Bonferroni":[208],"on":[210],"model-signal":[212],"pair":[213],"raises":[215],"non-vacuous":[219],"target":[220,267],"$0.75\u03c0_0$":[221],"0.05":[223],"0.16,":[225],"although":[226],"keeps":[229],"absolute":[230],"small.":[232],"Finally,":[233],"see":[237],"matters":[239],"after":[240],"deployment:":[241],"under":[242,257],"benchmark":[243],"shift":[244],"error":[246],"among":[247],"traces":[249],"tracks":[250],"new":[252],"task's":[253],"base":[254],"error,":[255],"best-of-$n$":[258],"against":[260],"rises":[264],"past":[265],"while":[268],"empirical":[270],"frequency":[272],"stays":[273],"below":[274],"$\u03b4$,":[275],"because":[276],"abstention":[277],"absorbs":[278],"failures.":[280]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-09T00:00:00"}