{"id":"https://openalex.org/W7214396159","doi":"https://doi.org/10.48550/arxiv.2609.29709","title":"Three Ways Classical Test Theory Can Mislead About LLM Judges","display_name":"Three Ways Classical Test Theory Can Mislead About LLM Judges","publication_year":2026,"publication_date":"2026-08-31","ids":{"openalex":"https://openalex.org/W7214396159","doi":"https://doi.org/10.48550/arxiv.2609.29709"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.29709","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.29709","pdf_url":"https://arxiv.org/pdf/2609.29709","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.29709","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5146185792","display_name":"Louis Yiven Zhu","orcid":"https://orcid.org/0009-0001-5579-0340","observed_orcids":["https://orcid.org/0009-0001-5579-0340"]},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Zhu, Louis Yiven","raw_affiliation_strings":["University of Oxford"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oxford","institution_ids":["https://openalex.org/I40120149"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5146185792"],"corresponding_institution_ids":["https://openalex.org/I40120149"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.5346999764442444,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.5346999764442444,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.03319999948143959,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.032099999487400055,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/classical-test-theory","display_name":"classical test theory","score":0.8169999718666077},{"id":"https://openalex.org/keywords/reliability-statistics","display_name":"reliability statistics","score":0.7099999785423279},{"id":"https://openalex.org/keywords/rubric-based-assessment","display_name":"rubric-based assessment","score":0.546999990940094},{"id":"https://openalex.org/keywords/decision-tables","display_name":"decision tables","score":0.328000009059906}],"concepts":[{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.6053000092506409},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5692999958992004},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5012999773025513},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.5005000233650208},{"id":"https://openalex.org/C140390113","wikidata":"https://www.wikidata.org/wiki/Q389391","display_name":"Classical test theory","level":4,"score":0.49540001153945923},{"id":"https://openalex.org/C89128539","wikidata":"https://www.wikidata.org/wiki/Q1949963","display_name":"Statistic","level":2,"score":0.48179998993873596},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.46799999475479126},{"id":"https://openalex.org/C40696583","wikidata":"https://www.wikidata.org/wiki/Q989120","display_name":"Type I and type II errors","level":2,"score":0.460099995136261},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.45969998836517334},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.44940000772476196},{"id":"https://openalex.org/C2777382242","wikidata":"https://www.wikidata.org/wiki/Q6017816","display_name":"Index (typography)","level":2,"score":0.42559999227523804},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.39800000190734863},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.38670000433921814},{"id":"https://openalex.org/C156639949","wikidata":"https://www.wikidata.org/wiki/Q13362737","display_name":"Item bank","level":4,"score":0.3758000135421753},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.36079999804496765},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.34790000319480896},{"id":"https://openalex.org/C169857963","wikidata":"https://www.wikidata.org/wiki/Q1461038","display_name":"Test statistic","level":3,"score":0.3416999876499176},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.3393999934196472},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.33090001344680786},{"id":"https://openalex.org/C19619285","wikidata":"https://www.wikidata.org/wiki/Q196372","display_name":"Observational error","level":2,"score":0.3172999918460846},{"id":"https://openalex.org/C97399411","wikidata":"https://www.wikidata.org/wiki/Q825367","display_name":"Coin flipping","level":2,"score":0.28459998965263367},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2815000116825104},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.27649998664855957},{"id":"https://openalex.org/C40969351","wikidata":"https://www.wikidata.org/wiki/Q3516228","display_name":"Word error rate","level":2,"score":0.2630999982357025},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.26010000705718994},{"id":"https://openalex.org/C77019957","wikidata":"https://www.wikidata.org/wiki/Q2689057","display_name":"Dependability","level":2,"score":0.25440001487731934},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.2531999945640564},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.25290000438690186}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.29709","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.29709","pdf_url":"https://arxiv.org/pdf/2609.29709","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.29709","doi":"https://doi.org/10.48550/arxiv.2609.29709","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.29709","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.29709","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.29709","pdf_url":"https://arxiv.org/pdf/2609.29709","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214396159.pdf","grobid_xml":"https://content.openalex.org/works/W7214396159.grobid-xml"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W2007434890","https://openalex.org/W2028944185","https://openalex.org/W2071168551","https://openalex.org/W2084403116","https://openalex.org/W2090964712","https://openalex.org/W2124173512","https://openalex.org/W4415797148","https://openalex.org/W7083308763","https://openalex.org/W7127383380","https://openalex.org/W7134821681","https://openalex.org/W7137017390","https://openalex.org/W7158270232","https://openalex.org/W7164913068"],"related_works":["https://openalex.org/W7127383380","https://openalex.org/W7134036278","https://openalex.org/W7165368332","https://openalex.org/W7154957730","https://openalex.org/W7149227717","https://openalex.org/W7168475147","https://openalex.org/W7164803637","https://openalex.org/W7220698693","https://openalex.org/W7204894163","https://openalex.org/W7131064212"],"abstract_inverted_index":{"Evaluations":[0],"that":[1,29,216,225],"use":[2],"a":[3,9,114,150,154,182,187,209,214,217,222,237],"large":[4],"language":[5],"model":[6,218],"(LLM)":[7],"as":[8,149,213,221],"judge":[10,46,176],"have":[11],"begun":[12],"to":[13,101,126,137,168],"borrow":[14],"reliability":[15,188,210,226],"statistics":[16,28],"from":[17,99,120,244],"classical":[18],"test":[19],"theory":[20],"and":[21,33,86,134,152,186,206,240],"its":[22,55],"extensions.":[23],"We":[24,235],"examine":[25],"three":[26],"such":[27],"need":[30],"one":[31,45,48],"administration":[32],"no":[34,51],"gold":[35,90,133,141,163,179],"labels.":[36],"None":[37],"of":[38,54,116,157,197,231],"them":[39],"can":[40],"isolate":[41],"the":[42,69,74,83,104,121,129,146,169,175,191,198,204,229,232],"judge,":[43,223],"because":[44],"under":[47],"prompt":[49,205],"supplies":[50],"variance":[52],"component":[53],"own.":[56],"Claude":[57],"Haiku":[58],"4.5":[59],"judged":[60],"210":[61],"constructed":[62],"short":[63],"answers":[64],"against":[65,132],"ten-element":[66],"checklists.":[67],"On":[68],"180":[70],"with":[71,165,228],"parsed":[72],"verdicts,":[73],"Kuder-Richardson":[75],"coefficient":[76],"(KR-20)":[77],"came":[78],"out":[79],"at":[80,103,153],"0.5223":[81],"on":[82,88,139],"judge's":[84,105,130,170],"verdicts":[85],"0.5231":[87],"error-free":[89,140,162],"verdicts.":[91,142],"In":[92],"simulation,":[93],"bank":[94],"design":[95],"alone":[96],"moves":[97,227],"KR-20":[98],"0.01":[100],"0.68":[102,138],"measured":[106],"4.72%":[107],"error":[108],"rate.":[109],"The":[110],"dependability":[111],"index":[112],"$\u03a6(\u03bb)$,":[113],"ratio":[115,189],"mean":[117],"squared":[118],"distances":[119],"pass":[122,155],"mark,":[123],"sits":[124],"0.22":[125],"0.38":[127],"below":[128,211],"accuracy":[131,144],"returns":[135],"0.54":[136],"Livingston-Lewis":[143],"treats":[145],"rubric":[147],"elements":[148,159],"sample,":[151],"mark":[156],"five":[158],"it":[160],"credits":[161],"scores":[164],"0.78,":[166],"close":[167],"0.81.":[171],"A":[172],"statement":[173],"about":[174],"therefore":[177],"needs":[178,190],"labels":[180],"or":[181],"varied":[183],"scorer":[184],"facet,":[185],"bank's":[192],"spread":[193,230],"beside":[194],"it.":[195],"One":[196],"four":[199,241],"closest":[200],"judge-evaluation":[201],"papers":[202],"varies":[203],"still":[207],"reads":[208],"0.7":[212],"sign":[215],"cannot":[219],"serve":[220],"although":[224],"samples":[233],"scored.":[234],"derive":[236],"decision":[238],"table":[239],"reporting":[242],"lines":[243],"these":[245],"two":[246],"rules.":[247]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-09-26T00:00:00"}