{"id":"https://openalex.org/W7225823635","doi":"https://doi.org/10.5281/zenodo.24132224","title":"Flagging stable classifier errors with an inventory-mediated LLM judgement: an exploratory study and a preregistered replication","display_name":"Flagging stable classifier errors with an inventory-mediated LLM judgement: an exploratory study and a preregistered replication","publication_year":2026,"publication_date":"2026-11-11","ids":{"openalex":"https://openalex.org/W7225823635","doi":"https://doi.org/10.5281/zenodo.24132224"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.24132224","doi":"https://doi.org/10.5281/zenodo.24132224","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24132224","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.24132224","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5155693284","display_name":"Mikto Miura","orcid":"https://orcid.org/0009-0000-4196-0508","observed_orcids":["https://orcid.org/0009-0000-4196-0508"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Miura, Mikoto","raw_affiliation_strings":["Independent Researcher"],"raw_orcid":"https://orcid.org/0009-0000-4196-0508","affiliations":[{"raw_affiliation_string":"Independent Researcher","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5137506836","display_name":"Yuusuke Harada","orcid":"https://orcid.org/0009-0006-7252-5115","observed_orcids":["https://orcid.org/0009-0006-7252-5115"]},"institutions":[{"id":"https://openalex.org/I113306721","display_name":"Hiroshima University","ror":"https://ror.org/03t78wx29","country_code":"JP","type":"education","lineage":["https://openalex.org/I113306721"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Harada, Yuusuke","raw_affiliation_strings":["Hiroshima University"],"raw_orcid":"https://orcid.org/0009-0006-7252-5115","affiliations":[{"raw_affiliation_string":"Hiroshima University","institution_ids":["https://openalex.org/I113306721"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5698000192642212,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5698000192642212,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3366999924182892,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.018200000748038292,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.8385999798774719},{"id":"https://openalex.org/C2780922921","wikidata":"https://www.wikidata.org/wiki/Q255189","display_name":"Paraphrase","level":2,"score":0.738099992275238},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.6384000182151794},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.5856999754905701},{"id":"https://openalex.org/C85973986","wikidata":"https://www.wikidata.org/wiki/Q1091731","display_name":"Exploratory research","level":2,"score":0.5515000224113464},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4796999990940094},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4742000102996826},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4634999930858612},{"id":"https://openalex.org/C2777548347","wikidata":"https://www.wikidata.org/wiki/Q5456937","display_name":"Flagging","level":2,"score":0.45719999074935913},{"id":"https://openalex.org/C157659113","wikidata":"https://www.wikidata.org/wiki/Q533822","display_name":"WordNet","level":2,"score":0.45019999146461487},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.4327999949455261},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.40220001339912415},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.40049999952316284},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.3743000030517578},{"id":"https://openalex.org/C103088060","wikidata":"https://www.wikidata.org/wiki/Q1062839","display_name":"Error detection and correction","level":2,"score":0.3467999994754791},{"id":"https://openalex.org/C162307627","wikidata":"https://www.wikidata.org/wiki/Q204833","display_name":"Enhanced Data Rates for GSM Evolution","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C75949130","wikidata":"https://www.wikidata.org/wiki/Q848010","display_name":"Database transaction","level":2,"score":0.2667999863624573}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.24132224","doi":"https://doi.org/10.5281/zenodo.24132224","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24132224","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.24132225","doi":"https://doi.org/10.5281/zenodo.24132225","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24132225","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.24132224","doi":"https://doi.org/10.5281/zenodo.24132224","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24132224","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Errors":[0],"that":[1,16,149],"a":[2,30,72,84,91,160,188],"classifier":[3,31],"repeats":[4],"under":[5,97,251,257],"paraphrase":[6,86],"and":[7,32,51,61,168,205,246,259,269,279],"reordering":[8],"of":[9,64,148,170,175,224],"its":[10,98],"input":[11],"(\"stable":[12],"errors\")":[13],"escape":[14],"detectors":[15],"look":[17],"for":[18,46,54],"inconsistency":[19],"among":[20],"the":[21,68,108,117,152,176,181,192,202,211,219,232,238,247],"model's":[22],"own":[23,99],"outputs.":[24],"We":[25,241],"asked":[26],"whether":[27],"disagreement":[28],"between":[29,166],"an":[33,38],"LLM":[34,74],"judgement":[35,110,164,234],"routed":[36],"through":[37],"explicit":[39],"external":[40],"structure":[41,69],"(a":[42],"WordNet":[43],"sense":[44],"inventory":[45,153],"word-in-context":[47,123],"judgements;":[48],"atomic":[49],"facts":[50],"label":[52],"definitions":[53],"NLI)":[55],"flags":[56],"such":[57],"errors":[58,103,113],"without":[59],"labels,":[60],"how":[62,146],"much":[63,147],"any":[65],"gain":[66],"needs":[67],"rather":[70],"than":[71,116,237],"second":[73,163],"judgement.":[75],"Study":[76,89,132,138,225],"1":[77],"(WiC":[78],"test,":[79,197],"ChaosNLI)":[80],"became":[81],"exploratory":[82],"when":[83],"preregistered":[85,92],"check":[87],"failed;":[88],"2,":[90],"replication":[93],"on":[94,143,195,206,210,221,264],"MCL-WiC,":[95],"stopped":[96],"rule":[100],"(91":[101],"stable":[102,112,228],"<":[104],"100).":[105],"Disagreement":[106],"with":[107],"inventory-mediated":[109],"flagged":[111],"more":[114],"often":[115],"classifier's":[118],"confidence":[119],"in":[120,131,137,273],"all":[121,144],"seven":[122],"configurations":[124],"(DeBERTa":[125],"\u00d7":[126],"Qwen:":[127],"0.206":[128],"[0.142,":[129],"0.268]":[130],"1,":[133],"0.268":[134],"[0.161,":[135],"0.374]":[136],"2),":[139],"but":[140],"not":[141,155],"consistently":[142],"errors;":[145],"advantage":[150],"needed":[151],"could":[154],"be":[156,261],"pinned":[157],"down,":[158],"because":[159],"working":[161],"unstructured":[162,182,239],"reached":[165],"25%":[167],"78%":[169],"it":[171,208],"(post":[172,213],"hoc).":[173,214],"Part":[174],"structured":[177,233],"judge's":[178,183],"edge":[179,193],"corrected":[180],"bias":[184],"towards":[185],"\"different\";":[186],"against":[187],"label-free":[189],"debiased":[190],"judge":[191],"remained":[194],"WiC":[196],"whose":[198],"reference":[199,220],"derives":[200],"from":[201],"same":[203],"inventory,":[204],"MCL-WiC":[207],"depended":[209],"correction":[212],"Two":[215],"independent":[216],"raters":[217],"confirmed":[218],"only":[222],"100":[223],"1's":[226],"208":[227],"errors.":[229],"On":[230],"NLI":[231],"was":[235],"worse":[236],"one.":[240],"report":[242],"both":[243],"preregistrations,":[244],"deviations":[245],"stop":[248],"decision.":[249],"Preprint,":[250],"review.":[252],"The":[253],"preregistrations":[254],"(OSF)":[255],"are":[256,271],"embargo":[258],"will":[260],"made":[262],"public":[263],"publication;":[265],"their":[266],"ids,":[267],"times":[268],"hashes":[270],"listed":[272],"Supplementary":[274],"S1.":[275],"Code,":[276],"configuration,":[277],"tables":[278],"aggregate":[280],"outputs:":[281],"doi:10.5281/zenodo.24132227.":[282]},"counts_by_year":[],"updated_date":"2026-11-12T12:28:57.059296","created_date":"2026-11-12T00:00:00"}