{"id":"https://openalex.org/W4403579486","doi":"https://doi.org/10.48550/arxiv.2410.13341","title":"Limits to scalable evaluation at the frontier: LLM as Judge won't beat twice the data","display_name":"Limits to scalable evaluation at the frontier: LLM as Judge won't beat twice the data","publication_year":2024,"publication_date":"2024-10-17","ids":{"openalex":"https://openalex.org/W4403579486","doi":"https://doi.org/10.48550/arxiv.2410.13341"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.13341","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2410.13341","pdf_url":"https://arxiv.org/pdf/2410.13341","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.13341","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5012103579","display_name":"Florian E. Dorner","orcid":"https://orcid.org/0009-0001-9258-6664","observed_orcids":["https://orcid.org/0009-0001-9258-6664"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dorner, Florian E.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093946918","display_name":"Vivian Y. Nastl","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nastl, Vivian Y.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5039915143","display_name":"Moritz Hardt","orcid":"https://orcid.org/0009-0000-7694-3038","observed_orcids":["https://orcid.org/0009-0000-7694-3038"]},"institutions":[{"id":"https://openalex.org/I149899117","display_name":"Max Planck Society","ror":"https://ror.org/01hhn8329","country_code":"DE","type":"nonprofit","lineage":["https://openalex.org/I149899117"]},{"id":"https://openalex.org/I4210135521","display_name":"Max Planck Institute for Intelligent Systems","ror":"https://ror.org/04fq9j139","country_code":"DE","type":"facility","lineage":["https://openalex.org/I149899117","https://openalex.org/I4210135521"]},{"id":"https://openalex.org/I4401726832","display_name":"T\u00fcbingen AI Center","ror":"https://ror.org/0107nyd78","country_code":"DE","type":"education","lineage":["https://openalex.org/I149899117","https://openalex.org/I4210135521","https://openalex.org/I4401726832","https://openalex.org/I8087733"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Hardt, Moritz","raw_affiliation_strings":["Max Planck Institute for Intelligent Systems , T\u00fcbingen","T\u00fcbingen AI Center"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Max Planck Institute for Intelligent Systems , T\u00fcbingen","institution_ids":["https://openalex.org/I149899117","https://openalex.org/I4210135521"]},{"raw_affiliation_string":"T\u00fcbingen AI Center","institution_ids":["https://openalex.org/I4401726832"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":31,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.44699999690055847,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.44699999690055847,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.1931000053882599,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.06710000336170197,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-as-a-judge","display_name":"LLM-as-a-judge","score":0.859000027179718},{"id":"https://openalex.org/keywords/bias-mitigation","display_name":"bias mitigation","score":0.8109999895095825},{"id":"https://openalex.org/keywords/theoretical-limits","display_name":"theoretical bounds","score":0.5950000286102295},{"id":"https://openalex.org/keywords/model-comparison","display_name":"model comparison","score":0.4650000035762787},{"id":"https://openalex.org/keywords/annotation-cost","display_name":"annotation cost","score":0.49900001287460327}],"concepts":[{"id":"https://openalex.org/C189809214","wikidata":"https://www.wikidata.org/wiki/Q829522","display_name":"Beat (acoustics)","level":2,"score":0.8247648477554321},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.7892246246337891},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.46628710627555847},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.36128708720207214},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.28918135166168213},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.14139670133590698},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.12465676665306091},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.11688670516014099},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.08765992522239685}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.13341","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2410.13341","pdf_url":"https://arxiv.org/pdf/2410.13341","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.13341","doi":"https://doi.org/10.48550/arxiv.2410.13341","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.13341","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.13341","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2410.13341","pdf_url":"https://arxiv.org/pdf/2410.13341","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403579486.pdf","grobid_xml":"https://content.openalex.org/works/W4403579486.grobid-xml"},"referenced_works_count":33,"referenced_works":["https://openalex.org/W2950446064","https://openalex.org/W3117956031","https://openalex.org/W4223908421","https://openalex.org/W4327810158","https://openalex.org/W4385571232","https://openalex.org/W4387994687","https://openalex.org/W4388327685","https://openalex.org/W4391632220","https://openalex.org/W4392616695","https://openalex.org/W4392781612","https://openalex.org/W4393946056","https://openalex.org/W4394708144","https://openalex.org/W4399455122","https://openalex.org/W4399694266","https://openalex.org/W4399836654","https://openalex.org/W4399837631","https://openalex.org/W4400104877","https://openalex.org/W4401042808","https://openalex.org/W4401043863","https://openalex.org/W4402466848","https://openalex.org/W4402670032","https://openalex.org/W4402670679","https://openalex.org/W4402671800","https://openalex.org/W4402699337","https://openalex.org/W4403006832","https://openalex.org/W4403662541","https://openalex.org/W4404782181","https://openalex.org/W4410537502","https://openalex.org/W4415795355","https://openalex.org/W4415795652","https://openalex.org/W4415797148","https://openalex.org/W4415797819","https://openalex.org/W6966639963"],"related_works":["https://openalex.org/W7156606905","https://openalex.org/W7223002623","https://openalex.org/W7214646924","https://openalex.org/W7211962755","https://openalex.org/W4391945622","https://openalex.org/W4404781910","https://openalex.org/W4404782878","https://openalex.org/W4412945123","https://openalex.org/W7128387938","https://openalex.org/W7165165314"],"abstract_inverted_index":{"High":[0],"quality":[1,79],"annotations":[2],"are":[3,160,180],"increasingly":[4],"a":[5,76,83],"bottleneck":[6],"in":[7,35,99,178],"the":[8,109,116,124,139,143,147,151,164,173,191],"explosively":[9],"growing":[10],"machine":[11],"learning":[12],"ecosystem.":[13],"Scalable":[14],"evaluation":[15,148],"methods":[16,200],"that":[17,58,107,159,172],"avoid":[18],"costly":[19,38],"annotation":[20],"have":[21],"therefore":[22],"become":[23],"an":[24,167],"important":[25],"research":[26],"ambition.":[27],"Many":[28],"hope":[29],"to":[30,40,70,81,138,154],"use":[31],"strong":[32],"existing":[33],"models":[34,50,158],"lieu":[36],"of":[37,48,66,86,127,142],"labels":[39,80,130],"provide":[41],"cheap":[42],"model":[43,61,87,202],"evaluations.":[44],"Unfortunately,":[45],"this":[46,90],"method":[47,121],"using":[49,75],"as":[51,56],"judges":[52],"introduces":[53],"biases,":[54],"such":[55,96],"self-preferencing,":[57],"can":[59,101,122],"distort":[60],"comparisons.":[62],"An":[63],"emerging":[64],"family":[65],"debiasing":[67,97,120,199],"tools":[68],"promises":[69],"fix":[71],"these":[72],"issues":[73],"by":[74,131],"few":[77],"high":[78],"debias":[82],"large":[84],"number":[85],"judgments.":[88],"In":[89],"paper,":[91],"we":[92,170],"study":[93],"how":[94],"far":[95],"methods,":[98],"principle,":[100],"go.":[102],"Our":[103,135],"main":[104],"result":[105,136],"shows":[106],"when":[108],"judge":[110],"is":[111,153],"no":[112,119],"more":[113,132,182],"accurate":[114],"than":[115,133,163,184],"evaluated":[117],"model,":[118],"decrease":[123],"required":[125],"amount":[126],"ground":[128],"truth":[129],"half.":[134],"speaks":[137],"severe":[140],"limitations":[141],"LLM-as-a-judge":[144],"paradigm":[145],"at":[146],"frontier":[149],"where":[150],"goal":[152],"assess":[155],"newly":[156],"released":[157],"possibly":[161],"better":[162],"judge.":[165],"Through":[166],"empirical":[168],"evaluation,":[169,203],"demonstrate":[171],"sample":[174],"size":[175],"savings":[176],"achievable":[177],"practice":[179],"even":[181],"modest":[183],"what":[185],"our":[186,193],"theoretical":[187],"limit":[188],"suggests.":[189],"Along":[190],"way,":[192],"work":[194],"provides":[195],"new":[196],"observations":[197],"about":[198],"for":[201,209],"and":[204],"points":[205],"out":[206],"promising":[207],"avenues":[208],"future":[210],"work.":[211]},"counts_by_year":[{"year":2026,"cited_by_count":15},{"year":2025,"cited_by_count":15},{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-10-20T00:00:00"}