{"id":"https://openalex.org/W7225147017","doi":"https://doi.org/10.5281/zenodo.23933494","title":"Matched Twins Show What an Evidence Contract Cannot Carry to a Verifier of ML-Experiment Claims","display_name":"Matched Twins Show What an Evidence Contract Cannot Carry to a Verifier of ML-Experiment Claims","publication_year":2026,"publication_date":"2026-11-04","ids":{"openalex":"https://openalex.org/W7225147017","doi":"https://doi.org/10.5281/zenodo.23933494"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.23933494","doi":"https://doi.org/10.5281/zenodo.23933494","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23933494","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.23933494","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5160531847","display_name":"Youngmin Ko","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ko, Youngmin","raw_affiliation_strings":["Pennsylvania State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pennsylvania State University","institution_ids":["https://openalex.org/I130769515"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5160531847"],"corresponding_institution_ids":["https://openalex.org/I130769515"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.14920000731945038,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.14920000731945038,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.13770000636577606,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.0568000003695488,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/trace-evidence","display_name":"trace evidence","score":0.5109999775886536},{"id":"https://openalex.org/keywords/blind-readers","display_name":"blind readers","score":0.3160000145435333},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.5239999890327454},{"id":"https://openalex.org/keywords/engineered-features","display_name":"engineered features","score":0.5080000162124634},{"id":"https://openalex.org/keywords/reproducibility-verification","display_name":"reproducibility verification","score":0.32600000500679016},{"id":"https://openalex.org/keywords/reproducibility-evaluation","display_name":"reproducibility evaluation","score":0.4300000071525574},{"id":"https://openalex.org/keywords/reproducibility-assessment","display_name":"reproducibility assessment","score":0.5419999957084656}],"concepts":[{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5419999957084656},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5360000133514404},{"id":"https://openalex.org/C2776299755","wikidata":"https://www.wikidata.org/wiki/Q432449","display_name":"Carry (investment)","level":2,"score":0.4788999855518341},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.46480000019073486},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.4481000006198883},{"id":"https://openalex.org/C2777026412","wikidata":"https://www.wikidata.org/wiki/Q2684591","display_name":"Statement (logic)","level":2,"score":0.44290000200271606},{"id":"https://openalex.org/C2777152325","wikidata":"https://www.wikidata.org/wiki/Q108163","display_name":"Proposition","level":2,"score":0.4345000088214874},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4239000082015991},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.4049000144004822},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.4025000035762787},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.40220001339912415},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3856000006198883},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.3781000077724457},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.36910000443458557},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3605000078678131},{"id":"https://openalex.org/C554936623","wikidata":"https://www.wikidata.org/wiki/Q199657","display_name":"Reading (process)","level":2,"score":0.3598000109195709},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.35429999232292175},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.34049999713897705},{"id":"https://openalex.org/C2778067643","wikidata":"https://www.wikidata.org/wiki/Q166507","display_name":"Interval (graph theory)","level":2,"score":0.3393999934196472},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.3337000012397766},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.32170000672340393},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.30570000410079956},{"id":"https://openalex.org/C59656382","wikidata":"https://www.wikidata.org/wiki/Q191536","display_name":"Conjunction (astronomy)","level":2,"score":0.30559998750686646},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.2913999855518341},{"id":"https://openalex.org/C39613435","wikidata":"https://www.wikidata.org/wiki/Q846677","display_name":"Remainder","level":2,"score":0.2849000096321106},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.28189998865127563},{"id":"https://openalex.org/C162838799","wikidata":"https://www.wikidata.org/wiki/Q596077","display_name":"Counterexample","level":2,"score":0.2809000015258789},{"id":"https://openalex.org/C27548731","wikidata":"https://www.wikidata.org/wiki/Q88272","display_name":"Investment (military)","level":3,"score":0.2777999937534332},{"id":"https://openalex.org/C64869954","wikidata":"https://www.wikidata.org/wiki/Q1859747","display_name":"False positive paradox","level":2,"score":0.27480000257492065},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.27300000190734863},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.2667999863624573},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.2646999955177307},{"id":"https://openalex.org/C119421448","wikidata":"https://www.wikidata.org/wiki/Q2268776","display_name":"Lying","level":2,"score":0.2567000091075897},{"id":"https://openalex.org/C85079727","wikidata":"https://www.wikidata.org/wiki/Q3560114","display_name":"Quarter (Canadian coin)","level":2,"score":0.2556999921798706},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.25450000166893005}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.23933494","doi":"https://doi.org/10.5281/zenodo.23933494","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23933494","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23933495","doi":"https://doi.org/10.5281/zenodo.23933495","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23933495","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.23933494","doi":"https://doi.org/10.5281/zenodo.23933494","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23933494","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7203446082","https://openalex.org/W7218604795","https://openalex.org/W7208745310","https://openalex.org/W7222948915","https://openalex.org/W7224127859","https://openalex.org/W7172548054","https://openalex.org/W7225107993","https://openalex.org/W7204970000","https://openalex.org/W7214483103","https://openalex.org/W7214954027"],"abstract_inverted_index":{"An":[0],"ML-engineering":[1],"agent":[2],"reports":[3],"that":[4,219,422,620,636,675,684,699],"adding":[5],"one":[6,200,203,229,635,782],"engineered":[7,230],"feature":[8,81],"improved":[9],"its":[10,240,280,318,383,407,411,740],"validation":[11],"score.":[12],"A":[13,192,340,419,718],"verifier":[14],"bounded":[15],"by":[16,655,715],"the":[17,22,25,31,36,41,54,79,89,91,109,121,140,148,165,170,183,205,214,235,244,267,270,275,283,290,303,325,336,343,368,373,387,396,403,414,424,431,435,449,453,464,469,475,479,485,489,494,506,513,538,547,550,562,569,573,580,583,607,622,625,634,646,668,749,753,761,775,786],"trace":[18],"evidence":[19],"(the":[20,98,332],"claim,":[21],"reporting":[23],"discipline,":[24],"outcome":[26],"of":[27,35,50,126,234,239,254,260,282,289,317,342,362,371,443,468,529,603,760],"plain":[28,476],"re-execution,":[29],"and":[30,61,73,106,153,157,174,224,238,259,288,294,305,324,328,335,398,409,413,482,505,545,561,586,694,781,797],"agent's":[32,206],"one-line":[33],"description":[34],"change)":[37],"must":[38],"decide":[39],"whether":[40,78],"gain":[42,477,481],"is":[43,82,108,196,628,643,713],"real.":[44],"We":[45,651],"construct":[46],"matched":[47,241,388,499,626],"twins:":[48],"pairs":[49,182,389],"executed":[51,706],"runs":[52],"on":[53,120,137,164,199,222,360,379,386,446,527,601,614,630,645,667,690,731,736],"same":[55,486,495,539,669],"task":[56,611,613],"whose":[57,66],"sentence,":[58,207,246],"model":[59,784],"types":[60],"budget":[62,263,276,285,515,575,608],"block":[63,277],"are":[64,777],"identical,":[65],"five-seed":[67],"mean":[68],"gains":[69],"agree":[70],"within":[71],"0.009,":[72],"which":[74,112,208],"differ":[75],"only":[76,187],"in":[77,124,458,566,746,789,792],"added":[80],"a":[83,188,255,440,459,473,518,542,598,640,663,671,677,681,691,695,738,793],"label":[84],"leak.":[85],"Scope-controlled":[86],"re-execution":[87,428,624],"separates":[88,147],"twins;":[90],"contract's":[92],"fields":[93],"carry":[94],"almost":[95],"no":[96,210],"signal":[97],"best":[99],"single":[100],"visible":[101],"field":[102,201,465],"reaches":[103,725],"AUC":[104],"0.67,":[105],"it":[107,356,589,638,642,732],"seed":[110],"variance,":[111],"we":[113,648,795],"did":[114,311,330],"not":[115,312,331,722,744],"manage":[116],"to":[117,168,178,393,399,493,522,556,559,592,688,758],"match:":[118],"larger":[119],"honest":[122,225,262,284,337,514,574],"twin":[123,154,242,500,697],"12":[125],"14":[127,181],"pairs).":[128],"Two":[129],"frontier":[130],"reader":[131,269,345,353,451,496,571,783],"models,":[132,780],"189":[133],"blind":[134,665],"readers":[135,776],"each":[136,250,602,660],"identical":[138],"prompts,":[139],"primary":[141,344,450,570],"reading":[142,463],"fixed":[143],"before":[144,307],"dispatch:":[145],"neither":[146],"pair.":[149],"One":[150],"flags":[151],"leak":[152,236,454],"at":[155,172,530,535,728],"11/14":[156,158,392],"(exact":[159],"p=1.0;":[160],"bootstrap":[161],"95%":[162],"interval":[163],"difference":[166],"-0.36":[167],"+0.36),":[169],"other":[171],"5/14":[173],"3/14":[175,394],"(p=0.69;":[176],"-0.21":[177],"+0.50);":[179],"with":[180,358,395,439,541,662,752],"design":[184],"can":[185,354],"exclude":[186],"gap":[189,763],"above":[190],"0.43.":[191],"second,":[193],"secondary":[194],"result":[195],"an":[197,261,299,351,723],"intervention":[198],"for":[202,249,553,612],"reader:":[204],"carries":[209,637],"truth":[211],"value":[212,600],"because":[213],"twins":[215],"share":[216],"it,":[217],"moves":[218],"reader's":[220],"verdicts":[221],"corrupt":[223],"traces":[226],"alike.":[227],"\"Add":[228],"feature\"":[231],"raises":[232],"rejection":[233,253,281],"(1/14\u219211/14)":[237],"(2/14\u219211/14);":[243],"tree-count":[245],"worded":[247],"truthfully":[248],"class,":[251],"lowers":[252],"starved":[256,291,507],"baseline":[257,292,508],"(18/18\u21924/18)":[258],"change":[264,286,516,576],"(9/22\u21923/22).":[265],"For":[266],"second":[268,548,581],"sentence":[271,333,397],"does":[272,743],"nothing;":[273],"removing":[274],"instead":[278],"drops":[279],"(22/22\u21926/22)":[287],"(17/18\u21929/18)":[293],"removes":[295],"every":[296,447,618,631],"unfair-baseline":[297],"call,":[298],"ablation":[300],"run":[301],"after":[302],"replication":[304,310],"registered":[306,369,420],"dispatch.":[308],"The":[309,594,703],"hold":[313],"as":[314,466,472,639],"registered:":[315],"two":[316,329,346,426,790],"four":[319,604],"readings":[320],"replicated":[321],"(twin":[322],"non-separability":[323],"legible":[326,437,492],"classes)":[327],"effect":[334,375,405],"false-positive":[338,519,673],"bound).":[339],"test-retest":[341],"days":[347,605],"later":[348],"bounds":[349],"what":[350],"unpinned":[352,778],"support:":[355],"agrees":[357],"itself":[359],"25":[361],"46":[363],"prompts":[364],"(kappa":[365],"0.15,":[366],"below":[367],"bar":[370],"0.6);":[372],"tree-sentence":[374],"reproduces":[376],"exactly":[377],"(18/18\u21924/18":[378],"both":[380,554],"days)":[381],"while":[382,511,606,733,764],"flag":[384],"rate":[385,674],"falls":[390,757],"from":[391,686],"0/14":[400,455],"without,":[401],"so":[402],"feature-sentence":[404],"keeps":[406],"direction":[408],"loses":[410],"size,":[412],"same-day":[415,460],"contrast":[416],"reads":[417],"indeterminate.":[418],"prediction":[421],"surfacing":[423],"labeler's":[425],"scope-controlled":[427],"deltas":[429],"inside":[430],"contract":[432,678],"would":[433],"make":[434],"pair":[436,490,595,627],"failed:":[438],"leak-free":[441],"delta":[442],"0.0":[444],"printed":[445],"leak,":[448],"flagged":[452],"against":[456],"7/14":[457],"control":[461,544],"(p=0.016),":[462],"confirmation":[467],"gain.":[470],"Re-encoded":[471],"comparison,":[474],"beside":[478],"scoped":[480,623],"their":[483],"gap,":[484],"re-executions":[487],"made":[488],"fully":[491],"(leak":[497],"14/14,":[498],"0/14,":[501],"p=1.2":[502],"\u00d7":[503],"10^-4)":[504],"too":[509],"(18/18),":[510],"turning":[512],"into":[517],"class":[520,739],"(3/22":[521],"17/22,":[523],"fourteen":[524],"called":[525,572],"unfair":[526,577,590],"gaps":[528],"most":[531],"0.01":[532],"or":[533],"none":[534],"all).":[536],"Replicated":[537],"day":[540,647],"fresh":[543],"under":[546,633],"reader,":[549],"separation":[551,698],"held":[552],"(14/14":[555],"0/14;":[557],"14/14":[558],"2/14)":[560],"magnitude":[563],"was":[564,676,700,773],"read":[565,582],"opposite":[567],"directions:":[568],"again":[578],"(18/22),":[579],"gap's":[584],"size":[585],"stopped":[587],"calling":[588],"(20/22":[591],"6/22).":[593],"rates":[596],"took":[597],"new":[599],"classes":[609],"reproduced":[610],"all":[615],"four.":[616],"Under":[617],"encoding":[619,679],"omits":[621],"non-separable":[629],"day;":[632],"comparison":[641],"separable":[644],"ran":[649],"it.":[650],"reached":[652],"this":[653],"measurement":[654],"catching":[656],"ourselves":[657],"three":[658],"times,":[659],"time":[661],"pre-registered":[664],"pilot":[666],"corpus:":[670],"0.909":[672],"defect,":[680],"leakage":[682],"recall":[683],"swung":[685],"0/21":[687],"15/21":[689],"corpus":[692,704],"artifact,":[693],"0.90-against-0.59":[696],"claim":[701],"magnitude.":[702],"(242":[705],"traces,":[707],"22":[708],"tasks,":[709],"11":[710],"scripted":[711],"proposers)":[712],"auto-labeled":[714],"counterfactual":[716],"re-execution.":[717],"deterministic":[719],"contract-bounded":[720],"stand-in,":[721],"LLM,":[724],"F1":[726],"0.82":[727],"precision":[729],"1.0":[730],"never":[734],"firing":[735],"leakage,":[737],"rule":[741],"set":[742],"encode;":[745],"candidate":[747],"selection":[748],"stand-in":[750],"filter":[751],"standard":[754],"claimed-gain":[755],"ranking":[756],"-0.02":[759],"oracle":[762],"re-executed":[765],"credit":[766],"closes":[767],"0.88.":[768],"No":[769],"funded":[770],"API":[771],"call":[772],"made;":[774],"session":[779],"violated":[785],"no-tools":[787],"instruction":[788],"dispatches":[791],"way":[794],"audit":[796],"disclose.":[798]},"counts_by_year":[],"updated_date":"2026-11-05T12:25:35.769452","created_date":"2026-11-04T00:00:00"}