{"id":"https://openalex.org/W7156814473","doi":"https://doi.org/10.48550/arxiv.2604.22891","title":"Quantifying and Mitigating Self-Preference Bias of LLM Judges","display_name":"Quantifying and Mitigating Self-Preference Bias of LLM Judges","publication_year":2026,"publication_date":"2026-04-24","ids":{"openalex":"https://openalex.org/W7156814473","doi":"https://doi.org/10.48550/arxiv.2604.22891"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.22891","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.22891","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.22891","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108545475","display_name":"Jinming Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Jinming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Hu, Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Zheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Qiu, Chuxian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiu, Chuxian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134805300","display_name":"Zhenyu Deng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deng, Zhenyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Jiao, Xinshan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiao, Xinshan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Zhou, Tao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Tao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.31709998846054077,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.31709998846054077,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10639","display_name":"Advanced Software Engineering Methodologies","score":0.13189999759197235,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.04179999977350235,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/conflation","display_name":"Conflation","score":0.7329000234603882},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.5777000188827515},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5275999903678894},{"id":"https://openalex.org/keywords/mainstream","display_name":"Mainstream","score":0.42910000681877136},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.40070000290870667},{"id":"https://openalex.org/keywords/cognitive-bias","display_name":"Cognitive bias","score":0.3912000060081482},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.39070001244544983},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.37459999322891235},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.3659000098705292}],"concepts":[{"id":"https://openalex.org/C130440534","wikidata":"https://www.wikidata.org/wiki/Q14946528","display_name":"Conflation","level":2,"score":0.7329000234603882},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.5777000188827515},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5530999898910522},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5275999903678894},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.42910000681877136},{"id":"https://openalex.org/C2777617010","wikidata":"https://www.wikidata.org/wiki/Q18957","display_name":"Mainstream","level":2,"score":0.42910000681877136},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.40070000290870667},{"id":"https://openalex.org/C189216375","wikidata":"https://www.wikidata.org/wiki/Q1127759","display_name":"Cognitive bias","level":3,"score":0.3912000060081482},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.39070001244544983},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.37459999322891235},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3659000098705292},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.34860000014305115},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3450999855995178},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3386000096797943},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.337799996137619},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.3237000107765198},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3224000036716461},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3084000051021576},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.2759000062942505},{"id":"https://openalex.org/C37381756","wikidata":"https://www.wikidata.org/wiki/Q20203288","display_name":"Representativeness heuristic","level":2,"score":0.27469998598098755},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.27469998598098755},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.271699994802475},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.26759999990463257},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.26269999146461487},{"id":"https://openalex.org/C44725695","wikidata":"https://www.wikidata.org/wiki/Q288156","display_name":"Normative","level":2,"score":0.2563000023365021},{"id":"https://openalex.org/C96608239","wikidata":"https://www.wikidata.org/wiki/Q1199823","display_name":"Statistical power","level":2,"score":0.2558000087738037},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.2556999921798706},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.2547000050544739},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.250900000333786}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.22891","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.22891","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.22891","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.22891","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.6943178176879883,"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"LLM-as-a-Judge":[0],"has":[1],"become":[2],"a":[3,41,88,145],"dominant":[4],"approach":[5,30],"in":[6,13,45,79,151],"automated":[7,90],"evaluation":[8,148],"systems,":[9],"playing":[10],"critical":[11],"roles":[12],"model":[14],"alignment,":[15],"leaderboard":[16],"construction,":[17],"quality":[18,105],"control,":[19],"and":[20,26,65,72,94],"so":[21],"on.":[22],"However,":[23],"the":[24],"scalability":[25],"trustworthiness":[27],"of":[28,101,110],"this":[29,84,141],"can":[31],"be":[32],"substantially":[33],"distorted":[34],"by":[35,158],"Self-Preference":[36],"Bias":[37],"(SPB),":[38],"which":[39,46,97,155],"is":[40],"directional":[42],"evaluative":[43,70],"deviation":[44],"LLMs":[47,124],"systematically":[48],"favor":[49],"or":[50,132],"disfavor":[51],"their":[52],"own":[53],"generated":[54],"outputs":[55],"during":[56],"evaluation.":[57],"Existing":[58],"measurements":[59],"rely":[60],"on":[61,160],"costly":[62],"human":[63,116],"annotations":[64],"conflate":[66],"generative":[67],"capability":[68],"with":[69,103,136],"stance,":[71],"thus":[73],"are":[74,129],"impractical":[75],"for":[76],"large-scale":[77],"deployment":[78],"real-world":[80],"systems.":[81],"To":[82,139],"address":[83],"issue,":[85],"we":[86,143],"introduce":[87],"fully":[89],"framework":[91],"to":[92],"quantifying":[93],"mitigating":[95],"SPB,":[96],"constructs":[98],"equal-quality":[99],"pairs":[100],"responses":[102],"negligible":[104],"differences,":[106],"enabling":[107],"statistical":[108],"disentanglement":[109],"discriminability":[111],"from":[112],"bias":[113],"propensity":[114],"without":[115],"gold":[117],"standards.":[118],"Empirical":[119],"analysis":[120],"across":[121],"20":[122],"mainstream":[123],"reveals":[125],"that":[126],"advanced":[127],"capabilities":[128],"often":[130],"uncorrelated,":[131],"even":[133],"negatively":[134],"correlated,":[135],"low":[137],"SPB.":[138],"mitigate":[140],"bias,":[142],"propose":[144],"structured":[146],"multi-dimensional":[147],"strategy":[149],"grounded":[150],"cognitive":[152],"load":[153],"decomposition,":[154],"reduces":[156],"SPB":[157],"31.5\\%":[159],"average.":[161]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-29T00:00:00"}