{"id":"https://openalex.org/W4415932859","doi":"https://doi.org/10.48550/arxiv.2504.03846","title":"Do LLM Evaluators Prefer Themselves for a Reason?","display_name":"Do LLM Evaluators Prefer Themselves for a Reason?","alternate_titles":[],"publication_year":2025,"publication_date":"2025-04-04","ids":{"openalex":"https://openalex.org/W4415932859","doi":"https://doi.org/10.48550/arxiv.2504.03846"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2504.03846","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.03846","pdf_url":"https://arxiv.org/pdf/2504.03846","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4415932859"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2504.03846","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136614647","display_name":"Wei-Lin Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chen, Wei-Lin","raw_affiliation_strings":["University of Virginia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Virginia","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120049692","display_name":"Zhepei Wei","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wei, Zhepei","raw_affiliation_strings":["University of Virginia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Virginia","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125705063","display_name":"Xinyu Zhu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhu, Xinyu","raw_affiliation_strings":["University of Virginia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Virginia","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120269825","display_name":"Shi Feng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I193531525","display_name":"George Washington University","ror":"https://ror.org/00y4zzh67","country_code":"US","type":"education","lineage":["https://openalex.org/I193531525"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Feng, Shi","raw_affiliation_strings":["George Washington University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"George Washington University","institution_ids":["https://openalex.org/I193531525"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5136077148","display_name":"Yu Meng","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Meng, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":56,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.725600004196167,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.725600004196167,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.15029999613761902,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.03189999982714653,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-assessment","display_name":"automated assessment","score":0.6959999799728394},{"id":"https://openalex.org/keywords/self-assessment","display_name":"self-assessment","score":0.4390000104904175},{"id":"https://openalex.org/keywords/model-size","display_name":"model size","score":0.4129999876022339},{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.8870000243186951},{"id":"https://openalex.org/keywords/code-generation","display_name":"code generation","score":0.8479999899864197},{"id":"https://openalex.org/keywords/factual-knowledge","display_name":"factual knowledge","score":0.9380000233650208},{"id":"https://openalex.org/keywords/chain-of-thought","display_name":"chain-of-thought","score":0.746999979019165},{"id":"https://openalex.org/keywords/reward-modeling","display_name":"reward modeling","score":0.47999998927116394},{"id":"https://openalex.org/keywords/self-refinement","display_name":"self-refinement","score":0.8960000276565552}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.6553999781608582},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.5914000272750854},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.5724999904632568},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5145000219345093},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.4869999885559082},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.48429998755455017},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.43320000171661377},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.38199999928474426},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3817000091075897},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3682999908924103},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.299699991941452},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2955999970436096},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.2838999927043915},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2793999910354614},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.26820001006126404},{"id":"https://openalex.org/C2779458634","wikidata":"https://www.wikidata.org/wiki/Q24963715","display_name":"Debiasing","level":2,"score":0.265500009059906}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2504.03846","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.03846","pdf_url":"https://arxiv.org/pdf/2504.03846","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2504.03846","doi":"https://doi.org/10.48550/arxiv.2504.03846","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2504.03846","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2504.03846","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.03846","pdf_url":"https://arxiv.org/pdf/2504.03846","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415932859.pdf","grobid_xml":"https://content.openalex.org/works/W4415932859.grobid-xml"},"referenced_works_count":62,"referenced_works":["https://openalex.org/W2084413241","https://openalex.org/W2798935874","https://openalex.org/W2901707424","https://openalex.org/W2980282514","https://openalex.org/W3082115681","https://openalex.org/W3083410900","https://openalex.org/W3134642945","https://openalex.org/W3177813494","https://openalex.org/W3195140297","https://openalex.org/W3200980294","https://openalex.org/W4221161695","https://openalex.org/W4282968607","https://openalex.org/W4308615640","https://openalex.org/W4311991106","https://openalex.org/W4385734161","https://openalex.org/W4385970045","https://openalex.org/W4386437475","https://openalex.org/W4386555476","https://openalex.org/W4387321091","https://openalex.org/W4387561528","https://openalex.org/W4387724705","https://openalex.org/W4387994739","https://openalex.org/W4389518896","https://openalex.org/W4392126192","https://openalex.org/W4392539324","https://openalex.org/W4395474395","https://openalex.org/W4396777837","https://openalex.org/W4399694266","https://openalex.org/W4399836654","https://openalex.org/W4400024829","https://openalex.org/W4401042538","https://openalex.org/W4401043020","https://openalex.org/W4401306886","https://openalex.org/W4401306979","https://openalex.org/W4402670032","https://openalex.org/W4402670561","https://openalex.org/W4402670679","https://openalex.org/W4402670898","https://openalex.org/W4402671190","https://openalex.org/W4402671800","https://openalex.org/W4402733601","https://openalex.org/W4403579486","https://openalex.org/W4403885225","https://openalex.org/W4404330874","https://openalex.org/W4404341075","https://openalex.org/W4404782181","https://openalex.org/W4404782878","https://openalex.org/W4405418995","https://openalex.org/W4405655184","https://openalex.org/W4407176798","https://openalex.org/W4407248359","https://openalex.org/W4414281281","https://openalex.org/W4415797064","https://openalex.org/W4415797148","https://openalex.org/W4416123545","https://openalex.org/W6929228989","https://openalex.org/W7080126089","https://openalex.org/W7133198585","https://openalex.org/W7133209743","https://openalex.org/W7133211296","https://openalex.org/W7133236450","https://openalex.org/W7133239970"],"related_works":["https://openalex.org/W7127170724","https://openalex.org/W7080126089","https://openalex.org/W4415797148","https://openalex.org/W4404341075","https://openalex.org/W7152387152","https://openalex.org/W4416037117","https://openalex.org/W7156814473","https://openalex.org/W7203909508","https://openalex.org/W7084136904","https://openalex.org/W4416176618"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,141,156,167,173,187],"(LLMs)":[3],"are":[4,194],"increasingly":[5],"used":[6],"as":[7,13,67,169,201],"automatic":[8],"evaluators":[9],"in":[10],"applications":[11],"such":[12,200],"benchmarking,":[14],"reward":[15],"modeling,":[16],"and":[17,40,171,218,241],"self-refinement.":[18],"Prior":[19],"work":[20],"highlights":[21],"a":[22,33,44,203,234],"potential":[23],"self-preference":[24,48,95,163,178],"bias":[25,179],"where":[26],"LLMs":[27],"favor":[28],"their":[29],"own":[30],"generated":[31],"responses,":[32],"tendency":[34],"often":[35],"intensifying":[36],"with":[37,150,216],"model":[38,132],"size":[39],"capability.":[41],"This":[42,110,184],"raises":[43],"critical":[45],"question:":[46],"Is":[47],"harmful,":[49],"or":[50],"does":[51],"it":[52],"simply":[53],"reflect":[54],"the":[55,210],"genuinely":[56,123],"higher-quality":[57],"outputs":[58],"of":[59,146,238],"stronger":[60,140,155,172,186],"models?":[61],"Answering":[62],"this":[63,91,147],"has":[64],"been":[65],"difficult":[66],"prior":[68],"works":[69],"mostly":[70,159],"relied":[71],"on":[72],"subjective":[73,229],"tasks":[74],"that":[75,82,105,220],"lack":[76],"an":[77],"objective":[78,107],"ground":[79],"truth,":[80],"meaning":[81],"either":[83],"preference":[84,148],"can":[85],"be":[86],"reasonably":[87],"justified.":[88],"To":[89],"address":[90],"ambiguity,":[92],"we":[93,214],"investigate":[94],"using":[96],"verifiable":[97,225],"benchmarks":[98,226],"(mathematical":[99],"reasoning,":[100],"factual":[101],"knowledge,":[102],"code":[103],"generation)":[104],"allow":[106],"ground-truth":[108],"assessment.":[109],"enables":[111],"us":[112],"to":[113,190,227],"distinguish":[114],"harmful":[115,177,211],"(favoring":[116,122],"objectively":[117,151],"worse":[118],"responses)":[119],"from":[120],"legitimate":[121],"superior":[124,152],"ones)":[125],"self-preference.":[126,212],"Our":[127],"large-scale":[128],"experiments":[129],"across":[130],"7":[131],"families":[133],"reveal":[134],"three":[135],"key":[136],"findings:":[137],"(1)":[138],"While":[139],"exhibit":[142],"greater":[143],"self-preference,":[144],"much":[145],"aligns":[149],"performance,":[153],"indicating":[154],"prefer":[157],"themselves":[158],"legitimately.":[160],"(2)":[161],"Harmful":[162],"persists":[164],"when":[165,180,192],"evaluator":[166],"err":[168],"generators,":[170],"display":[174],"more":[175,189,235],"pronounced":[176],"they":[181,193],"do":[182],"err.":[183],"suggests":[185],"struggle":[188],"recognize":[191],"wrong.":[195],"(3)":[196],"Inference-time":[197],"scaling":[198],"strategies,":[199],"generating":[202],"long":[204],"Chain-of-Thought":[205],"before":[206],"evaluation,":[207],"effectively":[208],"reduce":[209],"Additionally,":[213],"experiment":[215],"LMArena":[217],"show":[219],"our":[221],"findings":[222],"extend":[223],"beyond":[224],"real-world,":[228],"domains.":[230],"These":[231],"results":[232],"provide":[233],"nuanced":[236],"understanding":[237],"LLM-based":[239],"evaluation":[240],"practical":[242],"insights":[243],"for":[244],"improving":[245],"its":[246],"reliability.":[247]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4415932859","counts_by_year":[{"year":2026,"cited_by_count":41},{"year":2025,"cited_by_count":15}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2025-10-10T00:00:00"}
