{"id":"https://openalex.org/W7166835632","doi":"https://doi.org/10.18653/v1/2026.acl-long.706","title":"Mediocrity is the key for LLM as a Judge Anchor Selection","display_name":"Mediocrity is the key for LLM as a Judge Anchor Selection","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166835632","doi":"https://doi.org/10.18653/v1/2026.acl-long.706"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2026.acl-long.706","doi":"https://doi.org/10.18653/v1/2026.acl-long.706","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.706","pdf_url":"https://aclanthology.org/2026.acl-long.706.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2026.acl-long.706.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039944738","display_name":"Shachar Don-Yehiya","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]},{"id":"https://openalex.org/I197251160","display_name":"Hebrew University of Jerusalem","ror":"https://ror.org/03qxff017","country_code":"IL","type":"education","lineage":["https://openalex.org/I197251160"]}],"countries":["IL","US"],"is_corresponding":false,"raw_author_name":"Shachar Don-Yehiya","raw_affiliation_strings":["IBM Research ,","The Hebrew University of Jerusalem ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research ,","institution_ids":["https://openalex.org/I1341412227"]},{"raw_affiliation_string":"The Hebrew University of Jerusalem ,","institution_ids":["https://openalex.org/I197251160"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024163531","display_name":"Asaf Yehudai","orcid":"https://orcid.org/0000-0002-3743-7551","observed_orcids":["https://orcid.org/0000-0002-3743-7551"]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]},{"id":"https://openalex.org/I197251160","display_name":"Hebrew University of Jerusalem","ror":"https://ror.org/03qxff017","country_code":"IL","type":"education","lineage":["https://openalex.org/I197251160"]}],"countries":["IL","US"],"is_corresponding":false,"raw_author_name":"Asaf Yehudai","raw_affiliation_strings":["IBM Research ,","The Hebrew University of Jerusalem ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research ,","institution_ids":["https://openalex.org/I1341412227"]},{"raw_affiliation_string":"The Hebrew University of Jerusalem ,","institution_ids":["https://openalex.org/I197251160"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040286212","display_name":"Leshem Choshen","orcid":"https://orcid.org/0000-0002-0085-6496","observed_orcids":["https://orcid.org/0000-0002-0085-6496"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Leshem Choshen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5059068945","display_name":"Omri Abend","orcid":"https://orcid.org/0000-0003-4311-3876","observed_orcids":["https://orcid.org/0000-0003-4311-3876"]},"institutions":[{"id":"https://openalex.org/I197251160","display_name":"Hebrew University of Jerusalem","ror":"https://ror.org/03qxff017","country_code":"IL","type":"education","lineage":["https://openalex.org/I197251160"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Omri Abend","raw_affiliation_strings":["The Hebrew University of Jerusalem ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hebrew University of Jerusalem ,","institution_ids":["https://openalex.org/I197251160"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.45126891,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"15491","last_page":"15513"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8237000107765198,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8237000107765198,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.09529999643564224,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.009800000116229057,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-as-a-judge","display_name":"LLM-as-a-judge","score":0.9309999942779541},{"id":"https://openalex.org/keywords/anchor-selection","display_name":"anchor selection","score":0.984000027179718},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"pairwise comparison","score":0.6230000257492065},{"id":"https://openalex.org/keywords/power-analysis","display_name":"power analysis","score":0.7300000190734863},{"id":"https://openalex.org/keywords/alpacaeval","display_name":"AlpacaEval","score":0.7139999866485596},{"id":"https://openalex.org/keywords/effect-size","display_name":"effect size","score":0.5}],"concepts":[{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.7146999835968018},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.6478999853134155},{"id":"https://openalex.org/C117309347","wikidata":"https://www.wikidata.org/wiki/Q544697","display_name":"Mediocrity principle","level":2,"score":0.5720000267028809},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4009999930858612},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.32179999351501465},{"id":"https://openalex.org/C190253527","wikidata":"https://www.wikidata.org/wiki/Q295354","display_name":"Law and economics","level":1,"score":0.3183000087738037},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.289000004529953},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.28610000014305115},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.28529998660087585},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.2596000134944916}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.acl-long.706","doi":"https://doi.org/10.18653/v1/2026.acl-long.706","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.706","pdf_url":"https://aclanthology.org/2026.acl-long.706.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.acl-long.706","doi":"https://doi.org/10.18653/v1/2026.acl-long.706","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.706","pdf_url":"https://aclanthology.org/2026.acl-long.706.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166835632.pdf","grobid_xml":"https://content.openalex.org/works/W7166835632.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7213583947","https://openalex.org/W7172494924","https://openalex.org/W4399837631","https://openalex.org/W7204606853","https://openalex.org/W7137017390","https://openalex.org/W4416036694","https://openalex.org/W4416774401","https://openalex.org/W7083308763","https://openalex.org/W4412886854","https://openalex.org/W7167215375"],"abstract_inverted_index":{"The":[0],"\"LLM-as-a-judge\"":[1],"paradigm":[2],"has":[3],"become":[4],"a":[5,30,78,139,148],"standard":[6,161],"method":[7],"for":[8,156,166,180],"evaluating":[9,62],"open-ended":[10],"generation.To":[11],"address":[12],"the":[13,37,43,46,56,67,72,117,121,125,136],"quadratic":[14],"scalability":[15],"costs":[16],"of":[17,39,45,58,74,116,120,128,138],"pairwise":[18,167],"comparisons,":[19],"popular":[20],"benchmarks":[21],"like":[22],"Arena-Hard":[23],"and":[24,94,151,169,187],"AlpacaEval":[25],"compare":[26],"all":[27,109],"models":[28,175],"against":[29],"single":[31],"anchor.However,":[32],"despite":[33],"its":[34],"widespread":[35],"use,":[36],"impact":[38],"anchor":[40,59,75,80,91,129],"selection":[41,60,137],"on":[42,66],"reliability":[44],"results":[47],"remains":[48],"largely":[49],"unexplored.In":[50],"this":[51],"work,":[52],"we":[53,146,177],"systematically":[54],"investigate":[55],"effect":[57,126],"by":[61],"22":[63],"different":[64],"anchors":[65,102,183],"Arena-Hard-v2.0":[68],"dataset.We":[69],"find":[70],"that":[71,89,160],"choice":[73],"is":[76,133],"critical:":[77],"poor":[79,98],"can":[81],"dramatically":[82],"reduce":[83],"correlation":[84],"with":[85,143],"human":[86],"rankings.We":[87],"identify":[88],"common":[90],"choices":[92],"(best-performing":[93],"worst-performing":[95],"models)":[96],"make":[97],"anchors.Because":[99],"these":[100],"extreme":[101],"are":[103,113,164],"consistently":[104],"better":[105],"or":[106],"worse":[107],"than":[108],"other":[110],"models,":[111],"they":[112],"seldom":[114],"indicative":[115],"relative":[118],"ranking":[119],"models.We":[122],"further":[123],"quantify":[124],"size":[127],"selection,":[130],"showing":[131],"it":[132],"comparable":[134],"to":[135,171,184],"judge":[140],"model.We":[141],"conclude":[142],"actionable":[144],"recommendations.First,":[145],"conduct":[147],"power":[149],"analysis,":[150],"compute":[152],"sufficient":[153],"benchmark":[154,162],"sizes":[155,163],"anchor-based":[157],"evaluation,":[158],"finding":[159],"insufficient":[165],"evaluation":[168,189],"fail":[170],"distinguish":[172],"between":[173],"competitive":[174],"reliably.Second,":[176],"provide":[178],"guidelines":[179],"selecting":[181],"informative":[182],"ensure":[185],"reliable":[186],"efficient":[188],"practices.":[190]},"counts_by_year":[],"updated_date":"2026-10-29T09:56:32.521200","created_date":"2026-07-02T00:00:00"}