{"id":"https://openalex.org/W4416034556","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.805","title":"Curse of Knowledge: Your Guidance and Provided Knowledge are biasing LLM Judges in Complex Evaluation","display_name":"Curse of Knowledge: Your Guidance and Provided Knowledge are biasing LLM Judges in Complex Evaluation","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416034556","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.805"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.805","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.805","pdf_url":"https://aclanthology.org/2025.findings-emnlp.805.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-emnlp.805.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085948755","display_name":"W. P. Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Weiyuan Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100328102","display_name":"Xin Wang","orcid":"https://orcid.org/0000-0003-2686-466X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xintao Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Siyu Yuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Siyu Yuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011598794","display_name":"Rui Xu","orcid":"https://orcid.org/0000-0002-7157-3805"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rui Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021214395","display_name":"J Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiangjie Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100715389","display_name":"Qi Dong","orcid":"https://orcid.org/0000-0001-5683-8986"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qingqing Dong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yanghua Xiao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yanghua Xiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Deqing Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deqing Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"14900","last_page":"14924"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13583","display_name":"Jury Decision Making Processes","score":0.09260000288486481,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13583","display_name":"Jury Decision Making Processes","score":0.09260000288486481,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11477","display_name":"Epistemology, Ethics, and Metaphysics","score":0.08900000154972076,"subfield":{"id":"https://openalex.org/subfields/1211","display_name":"Philosophy"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12755","display_name":"Legal Education and Practice Innovations","score":0.03020000085234642,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/curse","display_name":"Curse","score":0.5600000023841858},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.2809000015258789},{"id":"https://openalex.org/keywords/power","display_name":"Power (physics)","score":0.2718000113964081},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.2621000111103058}],"concepts":[{"id":"https://openalex.org/C2780273121","wikidata":"https://www.wikidata.org/wiki/Q109411","display_name":"Curse","level":2,"score":0.5600000023841858},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.41040000319480896},{"id":"https://openalex.org/C190253527","wikidata":"https://www.wikidata.org/wiki/Q295354","display_name":"Law and economics","level":1,"score":0.3560999929904938},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.3418999910354614},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.3361999988555908},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.32429999113082886},{"id":"https://openalex.org/C118084267","wikidata":"https://www.wikidata.org/wiki/Q26110","display_name":"Positive economics","level":1,"score":0.32030001282691956},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.31949999928474426},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.2888000011444092},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.2849000096321106},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.28220000863075256},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.2809000015258789},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.2718000113964081},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2621000111103058},{"id":"https://openalex.org/C2777855551","wikidata":"https://www.wikidata.org/wiki/Q12310021","display_name":"Subject (documents)","level":2,"score":0.25859999656677246},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.2533000111579895},{"id":"https://openalex.org/C55587333","wikidata":"https://www.wikidata.org/wiki/Q1133029","display_name":"Engineering ethics","level":1,"score":0.2529999911785126}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-emnlp.805","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.805","pdf_url":"https://aclanthology.org/2025.findings-emnlp.805.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.805","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.805","pdf_url":"https://aclanthology.org/2025.findings-emnlp.805.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320335577","display_name":"Major Research Plan","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416034556.pdf","grobid_xml":"https://content.openalex.org/works/W4416034556.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"large":[1],"language":[2],"models":[3,91],"(LLMs)":[4],"grow":[5],"more":[6,132],"capable,":[7],"they":[8],"face":[9],"increasingly":[10],"diverse":[11],"and":[12,47,65,73,82,123,134,141,152,169,184,235,281,288,298,307,344,363,378,402],"complex":[13,40],"tasks,":[14],"making":[15],"reliable":[16],"evaluation":[17,126,136],"challenging.The":[18],"paradigm":[19],"of":[20,125,155,179,181,198],"LLMs":[21],"as":[22,26],"judges":[23],"has":[24],"emerged":[25],"a":[27,58,177,196],"scalable":[28],"solution,":[29],"yet":[30],"prior":[31],"work":[32],"primarily":[33],"focuses":[34],"on":[35],"simple":[36],"settings.Their":[37],"reliability":[38],"in":[39,293,325,331,339],"tasks-where":[41],"multi-faceted":[42],"rubrics,":[43],"unstructured":[44],"reference":[45,316,327],"answers,":[46],"nuanced":[48],"criteria":[49],"are":[50,238,296],"critical-remains":[51],"understudied.In":[52],"this":[53],"paper,":[54],"we":[55],"constructed":[56],"COM-PLEXEVAL,":[57],"challenge":[59],"benchmark":[60],"designed":[61],"to":[62,95,189,223,254,300,314,323,422,426],"systematically":[63,71],"expose":[64],"quantify":[66],"Auxiliary":[67],"Information":[68],"Induced":[69],"Biases.We":[70],"investigated":[72],"validated":[74],"6":[75],"previously":[76,409],"unexplored":[77],"biases":[78],"across":[79],"12":[80],"basic":[81],"3":[83],"advanced":[84],"scenarios.Key":[85],"findings":[86],"reveal:":[87],"(1)":[88],"all":[89,150],"evaluated":[90],"exhibit":[92],"significant":[93],"susceptibility":[94],"these":[96],"biases,":[97],"with":[98,102,285,356],"bias":[99],"magnitude":[100],"scaling":[101],"task":[103],"complexity;":[104],"(2)":[105],"notably,":[106],"Large":[107],"Reasoning":[108],"Models":[109],"(LRMs)":[110],"show":[111],"paradoxical":[112],"vulnerability.Our":[113],"in-depth":[114],"analysis":[115],"offers":[116],"crucial":[117],"insights":[118],"for":[119,131,241],"improving":[120],"the":[121,129,193,208,215,224,242,266,301,315,326,340,345,373],"accuracy":[122],"verifiability":[124],"signals,":[127],"paving":[128],"way":[130],"general":[133],"robust":[135],"models.*":[137],"Lacks":[138],"psychological":[139],"complexity":[140],"exhibits":[142],"rigid,":[143],"superficial":[144],"reactions":[145,321],"*":[146,161,175,186,231,252,261,274,290,303,318,353,390,398,404,412],"Directly":[147],"speaks":[148],"out":[149],"thoughts":[151,183],"feelings,":[153],"instead":[154],"using":[156],"subtext":[157],"-Type:":[158,172,228,248,270,386],"Persona":[159],"Coherence":[160],"Shows":[162,176,275,291,391],"inconsistent":[163],"or":[164,245,259,394,408,418],"rapidly":[165],"changing":[166],"personality":[167,287,311],"traits":[168,244,312],"emotional":[170],"patterns":[171],"Social":[173,351],"Interaction":[174],"lack":[178,197],"understanding":[180],"others'":[182,406],"feelings":[185],"Reacts":[187],"rigidly":[188],"others":[190],"without":[191],"considering":[192],"context.*":[194],"Demonstrate":[195],"appropriate":[199,240],"social":[200,364],"skills.\"\"\"},":[201],"\"Character":[202,346],"Fidelity\":":[203],"{":[204,369],"\"dimension_brief\":":[205,370],"\"How":[206,371],"well":[207,372],"characters":[209,358],"match":[210],"their":[211,286,360],"established":[212],"profiles":[213],"from":[214],"book\",":[216],"\"dimension_criteria\":":[217,382],"\"\"\"###":[218,383],"Character":[219,229],"Fidelity":[220],"(Only":[221],"apply":[222],"main":[225],"characters:":[226],"{major_characters})":[227],"Language":[230],"Uses":[232],"vocabulary,":[233],"expressions,":[234],"tone":[236],"that":[237,283,295],"not":[239],"characters'":[243],"social/educational":[246],"background":[247,258,289],"Knowledge":[249],"&":[250,272,350,388],"Background":[251],"Fails":[253],"demonstrate":[255,309],"character-specific":[256],"knowledge,":[257],"experiences":[260],"Includes":[262],"future":[263],"information":[264,411],"beyond":[265],"character's":[267],"current":[268],"stage":[269],"Personality":[271],"Behavior":[273],"emotions,":[276,306],"thoughts,":[277,305],"behaviors,":[278],"values,":[279],"beliefs,":[280],"decisions":[282],"conflict":[284],"interest":[292],"topics":[294],"uninteresting":[297],"unrelated":[299],"character":[302],"Character's":[304],"behaviors":[308],"contrasting":[310,320],"compared":[313,322],"conversation":[317,328,374],"Exhibits":[319],"those":[324],"if":[329],"situated":[330],"similar":[332],"contexts.(Such":[333],"flaws":[334],"should":[335],"be":[336],"counted":[337],"both":[338],"\"Storyline":[341,367],"Consistency\"":[342],"dimension":[343],"Fidelity\"":[347],"dimension.)-Type:":[348],"Relationship":[349],"Status":[352],"Interacts":[354],"inappropriately":[355],"other":[357],"regarding":[359],"background,":[361],"relationship":[362],"status\"\"\"":[365],"},":[366],"Quality\":":[368],"maintains":[375],"logical":[376],"consistency":[377],"narrative":[379],"quality":[380],"\",":[381],"Storyline":[384],"Quality":[385],"Flow":[387],"Progression":[389],"unnatural":[392],"progression":[393],"lacks":[395],"meaningful":[396],"developments":[397],"Dialogue":[399],"is":[400],"verbose":[401],"redundant":[403],"Repeats":[405],"viewpoints":[407],"mentioned":[410],"Mechanically":[413],"repeats":[414],"one's":[415],"own":[416],"words":[417],"phrases.More":[419],"repetitions":[420],"lead":[421],"higher":[423],"severity":[424],"(up":[425],"1":[427],"0).":[428]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-11-08T00:00:00"}