{"id":"https://openalex.org/W7204953855","doi":"https://doi.org/10.18653/2026.findings-emnlp.243","title":"Beyond Consensus: Downward Bias and Role Asymmetry in Multi-Agent LLM Judges for Subjective Evaluation","display_name":"Beyond Consensus: Downward Bias and Role Asymmetry in Multi-Agent LLM Judges for Subjective Evaluation","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7204953855","doi":"https://doi.org/10.18653/2026.findings-emnlp.243"},"language":"en","primary_location":{"id":"doi:10.18653/2026.findings-emnlp.243","doi":"https://doi.org/10.18653/2026.findings-emnlp.243","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.findings-emnlp.243","pdf_url":"https://aclanthology.org/2026.findings-emnlp.243.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2026","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://aclanthology.org/2026.findings-emnlp.243.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064731395","display_name":"Minsoo Song","orcid":"https://orcid.org/0000-0003-3823-4913","observed_orcids":["https://orcid.org/0000-0003-3823-4913"]},"institutions":[{"id":"https://openalex.org/I141371507","display_name":"Soongsil University","ror":"https://ror.org/017xnm587","country_code":"KR","type":"education","lineage":["https://openalex.org/I141371507"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Minsoo Song","raw_affiliation_strings":["Soongsil University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Soongsil University","institution_ids":["https://openalex.org/I141371507"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148873114","display_name":"Chanwoo Kim","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I141371507","display_name":"Soongsil University","ror":"https://ror.org/017xnm587","country_code":"KR","type":"education","lineage":["https://openalex.org/I141371507"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Chanwoo Kim","raw_affiliation_strings":["Soongsil University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Soongsil University","institution_ids":["https://openalex.org/I141371507"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080903430","display_name":"Sugyeong Eo","orcid":"https://orcid.org/0000-0002-8008-6160","observed_orcids":["https://orcid.org/0000-0002-8008-6160"]},"institutions":[{"id":"https://openalex.org/I193775966","display_name":"Yonsei University","ror":"https://ror.org/01wjejq96","country_code":"KR","type":"education","lineage":["https://openalex.org/I193775966"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Sugyeong Eo","raw_affiliation_strings":["Yonsei University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yonsei University","institution_ids":["https://openalex.org/I193775966"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5148873737","display_name":"Chanjun Park","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I141371507","display_name":"Soongsil University","ror":"https://ror.org/017xnm587","country_code":"KR","type":"education","lineage":["https://openalex.org/I141371507"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Chanjun Park","raw_affiliation_strings":["Soongsil University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Soongsil University","institution_ids":["https://openalex.org/I141371507"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.57163908,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"5116","last_page":"5133"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2029999941587448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2029999941587448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.17399999499320984,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.034699998795986176,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multi-agent-debate","display_name":"multi-agent debate","score":0.9890000224113464},{"id":"https://openalex.org/keywords/llm-as-a-judge","display_name":"LLM-as-a-judge","score":0.8220000267028809},{"id":"https://openalex.org/keywords/subjective-evaluation","display_name":"subjective evaluation","score":0.7689999938011169},{"id":"https://openalex.org/keywords/rubric-based-assessment","display_name":"rubric-based assessment","score":0.9369999766349792},{"id":"https://openalex.org/keywords/downward-bias","display_name":"downward bias","score":0.8709999918937683},{"id":"https://openalex.org/keywords/role-asymmetry","display_name":"role asymmetry","score":0.9710000157356262},{"id":"https://openalex.org/keywords/human-alignment","display_name":"human alignment","score":0.7990000247955322}],"concepts":[{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.5985999703407288},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5357000231742859},{"id":"https://openalex.org/C34130140","wikidata":"https://www.wikidata.org/wiki/Q1645406","display_name":"Midpoint","level":2,"score":0.45509999990463257},{"id":"https://openalex.org/C199776023","wikidata":"https://www.wikidata.org/wiki/Q202875","display_name":"Negotiation","level":2,"score":0.45399999618530273},{"id":"https://openalex.org/C38976095","wikidata":"https://www.wikidata.org/wiki/Q752641","display_name":"Asymmetry","level":2,"score":0.4440000057220459},{"id":"https://openalex.org/C151913843","wikidata":"https://www.wikidata.org/wiki/Q3454555","display_name":"Dominance (genetics)","level":3,"score":0.4399999976158142},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.42399999499320984},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.41290000081062317},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.40540000796318054},{"id":"https://openalex.org/C2777402240","wikidata":"https://www.wikidata.org/wiki/Q6783436","display_name":"Masking (illustration)","level":2,"score":0.34389999508857727},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3336000144481659},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.30090001225471497},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2978000044822693},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.2851000130176544},{"id":"https://openalex.org/C151243789","wikidata":"https://www.wikidata.org/wiki/Q17148646","display_name":"Multiple baseline design","level":3,"score":0.28290000557899475},{"id":"https://openalex.org/C169437150","wikidata":"https://www.wikidata.org/wiki/Q8458","display_name":"Human rights","level":2,"score":0.2680000066757202},{"id":"https://openalex.org/C138368954","wikidata":"https://www.wikidata.org/wiki/Q215028","display_name":"Peer review","level":2,"score":0.26570001244544983},{"id":"https://openalex.org/C87612349","wikidata":"https://www.wikidata.org/wiki/Q6008637","display_name":"In-group favoritism","level":4,"score":0.2632000148296356},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.2554999887943268},{"id":"https://openalex.org/C2777877512","wikidata":"https://www.wikidata.org/wiki/Q1116097","display_name":"Common ground","level":2,"score":0.2526000142097473}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/2026.findings-emnlp.243","doi":"https://doi.org/10.18653/2026.findings-emnlp.243","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.findings-emnlp.243","pdf_url":"https://aclanthology.org/2026.findings-emnlp.243.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2026","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2608.30373","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.30373","pdf_url":"https://arxiv.org/pdf/2608.30373","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.30373","doi":"https://doi.org/10.48550/arxiv.2608.30373","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.30373","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/2026.findings-emnlp.243","doi":"https://doi.org/10.18653/2026.findings-emnlp.243","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.findings-emnlp.243","pdf_url":"https://aclanthology.org/2026.findings-emnlp.243.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322006","display_name":"Ministry of Culture, Sports and Tourism","ror":"https://ror.org/02fkk6k65"},{"id":"https://openalex.org/F4320322120","display_name":"National Research Foundation of Korea","ror":"https://ror.org/013aysd81"},{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"},{"id":"https://openalex.org/F4320323890","display_name":"Korea Creative Content Agency","ror":"https://ror.org/036vyg793"},{"id":"https://openalex.org/F4320328359","display_name":"Ministry of Science and ICT, South Korea","ror":"https://ror.org/01wpjm123"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204953855.pdf","grobid_xml":"https://content.openalex.org/works/W7204953855.grobid-xml"},"referenced_works_count":5,"referenced_works":["https://openalex.org/W3045321166","https://openalex.org/W4391421129","https://openalex.org/W4404341075","https://openalex.org/W6947991961","https://openalex.org/W7166849922"],"related_works":["https://openalex.org/W4416034168","https://openalex.org/W4415308862","https://openalex.org/W7135066673","https://openalex.org/W7163367973","https://openalex.org/W6979242103","https://openalex.org/W7214926339","https://openalex.org/W7092114403","https://openalex.org/W7127581972","https://openalex.org/W7119510601","https://openalex.org/W7159661928"],"abstract_inverted_index":{"Multi-Agent":[0],"Debate":[1],"(MAD)":[2],"has":[3],"been":[4],"widely":[5],"adopted":[6],"to":[7,15,54,128],"improve":[8],"LLM-based":[9],"evaluation":[10,48],"by":[11],"prompting":[12,107],"multiple":[13],"agents":[14],"negotiate":[16],"and":[17,50,63,154,179],"reach":[18],"a":[19,38,42,114,119,201],"consensus.However,":[20],"for":[21,209],"subjective":[22,47,210],"rubric-based":[23],"scoring,":[24],"inter-agent":[25,175],"agreement":[26,192],"does":[27],"not":[28],"guarantee":[29],"alignment":[30,79,92],"with":[31],"human":[32,78,91,182,198],"judgments.In":[33],"this":[34,99,134],"paper,":[35],"we":[36],"compare":[37],"single-judge":[39,73],"baseline":[40,74,168],"against":[41],"consensus-based":[43],"MAD":[44,87,207],"protocol":[45],"on":[46,80,93,177],"tasks":[49],"design":[51],"three":[52],"ablations":[53,96],"isolate":[55],"the":[56,72,76,110,124,141,147,151,194],"impact":[57],"of":[58,150,196],"role":[59,106,117,162],"prompting,":[60],"multi-round":[61],"interaction,":[62],"explicit":[64],"score":[65,143],"sharing.Evaluations":[66],"across":[67,82],"six":[68,83],"LLMs":[69],"show":[70],"that":[71,98,123,133,186],"achieves":[75],"strongest":[77],"average":[81,178,181],"judge":[84,116],"models,":[85],"whereas":[86],"shows":[88],"degradation":[89],"in":[90,204],"both":[94],"tasks.Our":[95],"demonstrate":[97,185],"performance":[100],"drop":[101],"stems":[102],"primarily":[103],"from":[104],"asymmetric":[105],"rather":[108,157],"than":[109,158],"interaction":[111],"itself.Specifically,":[112],"assigning":[113],"strict":[115,153],"introduces":[118],"systematic":[120],"downward":[121],"bias":[122,135],"consensus":[125,142,188],"process":[126],"fails":[127],"correct.The":[129],"central":[130],"finding":[131],"is":[132],"reflects":[136],"strict-stance":[137],"dominance":[138],"beyond":[139,146],"averaging:":[140],"falls":[144],"well":[145],"arithmetic":[148],"midpoint":[149],"standalone":[152],"lenient":[155],"conditions,":[156],"averaging":[159],"them":[160],"out.Removing":[161],"asymmetry":[163],"(Symmetric":[164],"MAD)":[165],"largely":[166],"recovers":[167],"performance,":[169],"while":[170],"masking":[171],"peer":[172],"scores":[173],"widens":[174],"disagreement":[176],"worsens":[180],"alignment.These":[183],"findings":[184],"multi-agent":[187],"can":[189],"enforce":[190],"artificial":[191],"at":[193],"expense":[195],"true":[197],"alignment,":[199],"revealing":[200],"structural":[202],"limitation":[203],"consensus-style,":[205],"role-specialized":[206],"protocols":[208],"scoring.":[211]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-02T00:00:00"}