{"id":"https://openalex.org/W7214977693","doi":"https://doi.org/10.18653/2026.emnlp-main.2676","title":"Pair Difficulty Matters: Rethinking Pairwise LLM-as-a-Judge Evaluation and Consistency","display_name":"Pair Difficulty Matters: Rethinking Pairwise LLM-as-a-Judge Evaluation and Consistency","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7214977693","doi":"https://doi.org/10.18653/2026.emnlp-main.2676"},"language":"en","primary_location":{"id":"doi:10.18653/2026.emnlp-main.2676","doi":"https://doi.org/10.18653/2026.emnlp-main.2676","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.2676","pdf_url":"https://aclanthology.org/2026.emnlp-main.2676.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://aclanthology.org/2026.emnlp-main.2676.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135103517","display_name":"Bruno Brocai","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I223822909","display_name":"Heidelberg University","ror":"https://ror.org/038t36y30","country_code":"DE","type":"education","lineage":["https://openalex.org/I223822909"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Bruno Brocai","raw_affiliation_strings":["Heidelberg University German Department"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Heidelberg University German Department","institution_ids":["https://openalex.org/I223822909"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5153651043","display_name":"Maria Becker","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I223822909","display_name":"Heidelberg University","ror":"https://ror.org/038t36y30","country_code":"DE","type":"education","lineage":["https://openalex.org/I223822909"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Maria Becker","raw_affiliation_strings":["Heidelberg University German Department"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Heidelberg University German Department","institution_ids":["https://openalex.org/I223822909"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I223822909"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.78956449,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"60873","last_page":"60881"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.34940001368522644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.34940001368522644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.34130001068115234,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.11699999868869781,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"pairwise comparison","score":0.6029999852180481},{"id":"https://openalex.org/keywords/position-bias","display_name":"position bias","score":0.5870000123977661},{"id":"https://openalex.org/keywords/transitivity","display_name":"transitivity","score":0.9700000286102295},{"id":"https://openalex.org/keywords/bradley-terry-model","display_name":"Bradley-Terry model","score":0.8830000162124634},{"id":"https://openalex.org/keywords/text-ranking","display_name":"text ranking","score":0.5789999961853027},{"id":"https://openalex.org/keywords/human-evaluation","display_name":"human evaluation","score":0.3269999921321869}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.9369999766349792},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6606000065803528},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6344000101089478},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5566999912261963},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.5198000073432922},{"id":"https://openalex.org/C167729594","wikidata":"https://www.wikidata.org/wiki/Q17736","display_name":"Axiom","level":2,"score":0.43709999322891235},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.42969998717308044},{"id":"https://openalex.org/C98184364","wikidata":"https://www.wikidata.org/wiki/Q1780131","display_name":"Argument (complex analysis)","level":2,"score":0.39649999141693115},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.38850000500679016},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36169999837875366},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35019999742507935},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.34700000286102295},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3382999897003174},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.3345000147819519},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.33239999413490295},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.32739999890327454},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3197000026702881},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.31529998779296875},{"id":"https://openalex.org/C2778136018","wikidata":"https://www.wikidata.org/wiki/Q10350689","display_name":"Predictive power","level":2,"score":0.29989999532699585},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.29919999837875366},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.29840001463890076},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.2962000072002411},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2786000072956085},{"id":"https://openalex.org/C191399111","wikidata":"https://www.wikidata.org/wiki/Q64861","display_name":"Transitive relation","level":2,"score":0.2515000104904175}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/2026.emnlp-main.2676","doi":"https://doi.org/10.18653/2026.emnlp-main.2676","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.2676","pdf_url":"https://aclanthology.org/2026.emnlp-main.2676.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2609.37577","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.37577","pdf_url":"https://arxiv.org/pdf/2609.37577","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.37577","doi":"https://doi.org/10.48550/arxiv.2609.37577","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.37577","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/2026.emnlp-main.2676","doi":"https://doi.org/10.18653/2026.emnlp-main.2676","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.2676","pdf_url":"https://aclanthology.org/2026.emnlp-main.2676.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214977693.pdf","grobid_xml":"https://content.openalex.org/works/W7214977693.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7220134105","https://openalex.org/W7126190339","https://openalex.org/W7129012486","https://openalex.org/W7167215375","https://openalex.org/W7214003007","https://openalex.org/W7211962755","https://openalex.org/W4414790729","https://openalex.org/W4412886854","https://openalex.org/W7127269302","https://openalex.org/W7130581838"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Model":[2],"judges":[3,45],"are":[4],"widely":[5],"used":[6],"to":[7,86],"rank":[8],"texts":[9],"and":[10,16,28,38,81,104,111,126],"text-generating":[11],"systems":[12],"through":[13],"pairwise":[14,29,67],"comparison,":[15],"their":[17,127],"reliability":[18],"is":[19,61,71,78],"typically":[20],"assessed":[21,138],"via":[22],"three":[23],"proxies:":[24],"position":[25],"bias,":[26],"transitivity,":[27],"agreement":[30],"(self-or":[31],"human-labeled).Because":[32],"these":[33],"proxies":[34,117],"drive":[35],"judge":[36],"selection":[37],"benchmarking,":[39],"a":[40,108],"substantial":[41],"literature":[42],"reporting":[43],"that":[44],"perform":[46],"poorly":[47],"on":[48,112,139],"them":[49],"risks":[50],"steering":[51],"practitioners":[52],"away":[53],"from":[54],"otherwise":[55],"capable":[56],"evaluators.We":[57],"argue":[58],"this":[59,102],"assessment":[60],"misleading.Under":[62],"the":[63,87,93,99,116,132],"Bradley-Terry":[64],"geometry":[65],"underlying":[66],"aggregation,":[68],"each":[69],"proxy":[70],"dominated":[72],"by":[73],"close-rank-gap":[74],"pairs,":[75],"where":[76],"inconsistency":[77],"information-theoretically":[79],"expected":[80],"individual":[82],"verdicts":[83],"contribute":[84],"little":[85],"aggregate":[88],"ranking;":[89],"far-gap":[90,133],"pairs":[91],"carry":[92],"ranking":[94,122],"signal":[95],"but":[96],"barely":[97],"move":[98],"proxies.We":[100],"formalize":[101],"argument":[103],"validate":[105],"it":[106],"in":[107,131],"controlled":[109],"simulation":[110],"two":[113],"human-rated":[114],"corpora:":[115],"correlate":[118],"only":[119],"weakly":[120],"with":[121],"accuracy":[123],"against":[124,143],"gold,":[125],"predictive":[128],"component":[129],"concentrates":[130],"regime.Judges":[134],"should":[135],"therefore":[136],"be":[137],"rank-gap-conditional":[140],"metrics,":[141],"ideally":[142],"human":[144],"rankings.Code":[145],"at":[146],"https://github.com/brunobrocai/PairDifficulty.":[147]},"counts_by_year":[],"updated_date":"2026-10-31T12:09:11.096373","created_date":"2026-10-01T00:00:00"}