{"id":"https://openalex.org/W7214756010","doi":"https://doi.org/10.48550/arxiv.2609.31857","title":"LLM Judge Validation Under Sparse Overlap: From Inference to Design","display_name":"LLM Judge Validation Under Sparse Overlap: From Inference to Design","publication_year":2026,"publication_date":"2026-09-25","ids":{"openalex":"https://openalex.org/W7214756010","doi":"https://doi.org/10.48550/arxiv.2609.31857"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.31857","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.31857","pdf_url":"https://arxiv.org/pdf/2609.31857","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.31857","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5153411960","display_name":"Junxuan Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li, Junxuan","raw_affiliation_strings":["Adobe Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Adobe Research","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042545880","display_name":"Arko Provo Mukherjee","orcid":"https://orcid.org/0000-0003-3342-7804","observed_orcids":["https://orcid.org/0000-0003-3342-7804"]},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mukherjee, Arko","raw_affiliation_strings":["Adobe Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Adobe Research","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047754690","display_name":"Soumyabrata Pal","orcid":"https://orcid.org/0000-0003-2949-3761","observed_orcids":["https://orcid.org/0000-0003-2949-3761"]},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pal, Soumyabrata","raw_affiliation_strings":["Adobe Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Adobe Research","institution_ids":["https://openalex.org/I1306409833"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1306409833"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.24269999563694,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.24269999563694,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.2093999981880188,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.05130000039935112,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-as-a-judge","display_name":"LLM-as-a-judge","score":0.9580000042915344},{"id":"https://openalex.org/keywords/human-llm-agreement","display_name":"human-LLM agreement","score":0.656000018119812},{"id":"https://openalex.org/keywords/annotation-budget","display_name":"annotation budget","score":0.6840000152587891},{"id":"https://openalex.org/keywords/stratified-sampling","display_name":"stratified sampling","score":0.9549999833106995},{"id":"https://openalex.org/keywords/visual-assessment","display_name":"visual assessment","score":0.6340000033378601},{"id":"https://openalex.org/keywords/causal-reasoning","display_name":"causal reasoning","score":0.9950000047683716},{"id":"https://openalex.org/keywords/summarization","display_name":"summarization","score":0.9750000238418579}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.7228999733924866},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7113999724388123},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5849000215530396},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.515500009059906},{"id":"https://openalex.org/C158600405","wikidata":"https://www.wikidata.org/wiki/Q5054566","display_name":"Causal inference","level":2,"score":0.4406999945640564},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4194999933242798},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.40299999713897705},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40119999647140503},{"id":"https://openalex.org/C49898467","wikidata":"https://www.wikidata.org/wiki/Q1517706","display_name":"Stratified sampling","level":2,"score":0.3617999851703644},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.3465999960899353},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3384999930858612},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.3384000062942505},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.32339999079704285},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3190000057220459},{"id":"https://openalex.org/C917703","wikidata":"https://www.wikidata.org/wiki/Q7239668","display_name":"Predictive inference","level":5,"score":0.30399999022483826},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2930000126361847},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.29269999265670776},{"id":"https://openalex.org/C75373757","wikidata":"https://www.wikidata.org/wiki/Q7410160","display_name":"Sampling design","level":3,"score":0.28700000047683716},{"id":"https://openalex.org/C142853389","wikidata":"https://www.wikidata.org/wiki/Q744778","display_name":"Association (psychology)","level":2,"score":0.26669999957084656},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.25290000438690186}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.31857","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.31857","pdf_url":"https://arxiv.org/pdf/2609.31857","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.31857","doi":"https://doi.org/10.48550/arxiv.2609.31857","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.31857","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.31857","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.31857","pdf_url":"https://arxiv.org/pdf/2609.31857","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214756010.pdf","grobid_xml":"https://content.openalex.org/works/W7214756010.grobid-xml"},"referenced_works_count":3,"referenced_works":["https://openalex.org/W2346705140","https://openalex.org/W4405253355","https://openalex.org/W7160161636"],"related_works":["https://openalex.org/W4399837631","https://openalex.org/W4417070592","https://openalex.org/W7165368332","https://openalex.org/W7167897301","https://openalex.org/W7128491796","https://openalex.org/W7203918292","https://openalex.org/W7213999700","https://openalex.org/W4387994739","https://openalex.org/W7090848549","https://openalex.org/W7203791510"],"abstract_inverted_index":{"Validating":[0],"an":[1],"LLM-as-a-judge":[2],"requires":[3],"estimating":[4],"its":[5],"agreement":[6],"with":[7],"humans,":[8],"yet":[9],"annotation":[10],"budgets":[11],"rarely":[12],"allow":[13],"every":[14],"item":[15],"to":[16,95],"be":[17],"multiply":[18],"labeled.":[19],"We":[20,102],"prove":[21],"that":[22],"this":[23],"overlap":[24,61],"sparsity":[25],"is":[26,54],"the":[27,43,47],"first-order":[28],"determinant":[29],"of":[30,45],"wrong":[31,48],"deployment":[32],"decisions:":[33],"at":[34],"5%":[35],"pairwise":[36],"overlap,":[37],"wrong-decision":[38],"rates":[39,93],"reach":[40],"25%":[41],"and":[42,63,117],"probability":[44],"selecting":[46],"best":[49],"judge":[50],"among":[51],"ten":[52],"candidates":[53],"65%.":[55],"The":[56],"two":[57],"actionable":[58],"levers":[59],"are":[60,100],"quantity":[62],"allocation.":[64],"For":[65,85],"quantity,":[66],"we":[67],"derive":[68],"a":[69,87],"minimum-overlap":[70],"formula":[71],"showing":[72],"$\u03c1\\geq":[73],"0.25$":[74],"suffices":[75],"for":[76],"non-borderline":[77],"judges":[78,107],"while":[79],"borderline":[80],"cases":[81],"remain":[82],"fundamentally":[83],"hard.":[84],"allocation,":[86],"zero-cost":[88],"stratified":[89],"scheme":[90],"halves":[91],"false-rejection":[92],"relative":[94],"random":[96],"sampling":[97],"when":[98],"strata":[99],"informative.":[101],"validate":[103],"on":[104],"10":[105],"LLM":[106],"across":[108],"four":[109],"evaluation":[110],"matrices":[111],"spanning":[112],"visual":[113],"assessment,":[114],"causal":[115],"reasoning,":[116],"summarization.":[118]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-09-30T00:00:00"}