{"id":"https://openalex.org/W7167228722","doi":"https://doi.org/10.48550/arxiv.2607.02104","title":"When Can You Debias an LLM Judge? Identifiability Limits, a Test, and Designs for Top-k Ranking","display_name":"When Can You Debias an LLM Judge? Identifiability Limits, a Test, and Designs for Top-k Ranking","publication_year":2026,"publication_date":"2026-07-02","ids":{"openalex":"https://openalex.org/W7167228722","doi":"https://doi.org/10.48550/arxiv.2607.02104"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.02104","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.02104","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.02104","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111626467","display_name":"J Q Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Jian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140007387","display_name":"Delu Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Delu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139956236","display_name":"John Paisley","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paisley, John","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139970870","display_name":"Qibin Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Qibin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.32510000467300415,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.32510000467300415,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09390000253915787,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.05810000002384186,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ask-price","display_name":"Ask price","score":0.6583999991416931},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.5964000225067139},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5895000100135803},{"id":"https://openalex.org/keywords/bayesian-inference","display_name":"Bayesian inference","score":0.5242999792098999},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.49309998750686646},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.39809998869895935},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.38029998540878296},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.36500000953674316},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.3594000041484833}],"concepts":[{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.6583999991416931},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.5964000225067139},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5895000100135803},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5282999873161316},{"id":"https://openalex.org/C160234255","wikidata":"https://www.wikidata.org/wiki/Q812535","display_name":"Bayesian inference","level":3,"score":0.5242999792098999},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.49309998750686646},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4611999988555908},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4602000117301941},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.39809998869895935},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.396699994802475},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.38029998540878296},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.36500000953674316},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3594000041484833},{"id":"https://openalex.org/C119043178","wikidata":"https://www.wikidata.org/wiki/Q320723","display_name":"Covariate","level":2,"score":0.31619998812675476},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3091000020503998},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.30790001153945923},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.30149999260902405},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.2985000014305115},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.2930000126361847},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2928999960422516},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.28029999136924744},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.27480000257492065},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C162376815","wikidata":"https://www.wikidata.org/wiki/Q2158281","display_name":"Frequentist inference","level":4,"score":0.2680000066757202},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.2678000032901764},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.2630999982357025},{"id":"https://openalex.org/C158600405","wikidata":"https://www.wikidata.org/wiki/Q5054566","display_name":"Causal inference","level":2,"score":0.2621000111103058},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2603999972343445},{"id":"https://openalex.org/C2776325391","wikidata":"https://www.wikidata.org/wiki/Q6917865","display_name":"Motivated reasoning","level":3,"score":0.2581000030040741},{"id":"https://openalex.org/C101112237","wikidata":"https://www.wikidata.org/wiki/Q4874481","display_name":"Bayesian statistics","level":4,"score":0.2567000091075897}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.02104","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.02104","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.02104","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.02104","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"are":[4],"increasingly":[5],"used":[6],"as":[7,46],"cheap,":[8],"scalable":[9],"judges":[10,18,205,221],"that":[11,116,174],"compare":[12],"candidate":[13],"outputs":[14],"pairwise.":[15],"Because":[16],"such":[17],"prefer":[19],"verbose":[20],"or":[21],"well-formatted":[22],"answers,":[23],"the":[24,38,48,58,67,71,80,91,118,127,137,155,236,242],"natural":[25],"fix":[26],"is":[27,51,60,73,88,99,112,123,132,152,207,223],"to":[28,32,75],"add":[29],"bias":[30,39,206],"covariates":[31],"a":[33,102,106,133,146,171,189,197,224],"Bradley--Terry":[34],"model":[35,157],"and":[36,57,78,161,165,196,209,222,233],"estimate":[37],"away.":[40],"We":[41,167],"show":[42],"this":[43],"cannot":[44],"work":[45],"advertised:":[47],"quality/bias":[49],"split":[50],"\\emph{not":[52],"identified}":[53],"by":[54,90,215],"pairwise":[55],"comparisons,":[56],"failure":[59],"exact":[61],"--":[62,130,139],"across":[63],"$48$":[64],"real":[65,203],"judge-pools":[66],"profile":[68],"likelihood":[69],"over":[70,235],"coefficient":[72],"flat":[74],"$\\mathbf{0.0000}$":[76],"\\textbf{nats},":[77],"scaling":[79],"comparisons":[81],"$26\\times$":[82],"buys":[83],"none.":[84],"A":[85],"``debiased''":[86],"score":[87],"selected":[89],"prior,":[92],"not":[93,101],"recovered":[94],"from":[95],"data.":[96],"Our":[97],"contribution":[98],"accordingly":[100],"better":[103],"estimator":[104],"but":[105],"characterization":[107],"of":[108],"\\emph{when":[109],"prior-based":[110],"correction":[111,211],"justified},":[113],"plus":[114],"designs":[115],"supply":[117],"missing":[119],"information":[120],"when":[121],"it":[122],"not.":[124],"The":[125],"assumption":[126],"prior":[128],"encodes":[129],"quality":[131],"priori":[134],"uncorrelated":[135],"with":[136],"covariate":[138],"pays":[140],"only":[141],"while":[142],"$\\mathrm{corr}(\u03b8,x)$":[143],"stays":[144],"below":[145],"crossing":[147],"point":[148],"(configuration-dependent,":[149],"$0.22$--$0.60$),":[150],"which":[151],"what":[153],"makes":[154],"same":[156],"help":[158],"on":[159,163,217,226],"LLMBar":[160],"hurt":[162],"SummEval":[164],"Nectar.":[166],"give":[168],"two":[169],"escapes:":[170],"\\textbf{trusted-anchor":[172],"gate}":[173],"decides":[175],"per":[176],"(judge,":[177],"covariate,":[178],"task)":[179],"(no":[180],"false":[181],"enables":[182],"in":[183],"$6{,}000$":[184],"decisions":[185],"at":[186,194],"$K\\ge10$":[187],"anchors,":[188],"rate":[190],"our":[191],"sample":[192],"bounds":[193],"$\\le6\\%$),":[195],"\\textbf{paired":[198],"rendering":[199],"design}.":[200],"Across":[201],"fifteen":[202],"LLM":[204],"heterogeneous":[208],"capability-dependent:":[210],"improves":[212],"\\topk{}":[213],"recall":[214],"$0.20$--$0.32$":[216],"five":[218],"biased-but-competent":[219],"cheap":[220],"no-op":[225],"frontier":[227],"ones":[228],"(Spearman":[229],"$\u03c1{=}{-}0.84$":[230],"between":[231],"competence":[232],"gain":[234],"$14$":[237],"competent":[238],"judges,":[239],"$p{<}10^{-3}$),":[240],"concentrating":[241],"benefit":[243],"where":[244],"at-scale":[245],"evaluation":[246],"happens.":[247]},"counts_by_year":[],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2026-07-04T00:00:00"}