{"id":"https://openalex.org/W7168361147","doi":"https://doi.org/10.1145/3805712.3809905","title":"When LLM Judges Inflate Scores: Exploring Overrating in Relevance Assessment","display_name":"When LLM Judges Inflate Scores: Exploring Overrating in Relevance Assessment","publication_year":2026,"publication_date":"2026-07-15","ids":{"openalex":"https://openalex.org/W7168361147","doi":"https://doi.org/10.1145/3805712.3809905"},"language":null,"primary_location":{"id":"doi:10.1145/3805712.3809905","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3809905","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805712.3809905","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140939104","display_name":"Chuting Yu","orcid":"https://orcid.org/0009-0008-5150-3843"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Chuting Yu","raw_affiliation_strings":["The University of Queensland, Brisbane, QLD, Australia"],"raw_orcid":"https://orcid.org/0009-0008-5150-3843","affiliations":[{"raw_affiliation_string":"The University of Queensland, Brisbane, QLD, Australia","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140949682","display_name":"Hang Li","orcid":"https://orcid.org/0000-0002-5317-7227"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Hang Li","raw_affiliation_strings":["The University of Queensland, Brisbane, QLD, Australia"],"raw_orcid":"https://orcid.org/0000-0002-5317-7227","affiliations":[{"raw_affiliation_string":"The University of Queensland, Brisbane, QLD, Australia","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140804823","display_name":"Guido Zuccon","orcid":"https://orcid.org/0000-0003-0271-5563"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Guido Zuccon","raw_affiliation_strings":["The University of Queensland, Brisbane, QLD, Australia"],"raw_orcid":"https://orcid.org/0000-0003-0271-5563","affiliations":[{"raw_affiliation_string":"The University of Queensland, Brisbane, QLD, Australia","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140780162","display_name":"Joel Mackenzie","orcid":"https://orcid.org/0000-0001-7992-4633"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Joel Mackenzie","raw_affiliation_strings":["The University of Queensland, Brisbane, QLD, Australia"],"raw_orcid":"https://orcid.org/0000-0001-7992-4633","affiliations":[{"raw_affiliation_string":"The University of Queensland, Brisbane, QLD, Australia","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5140894421","display_name":"Teerapong Leelanupab","orcid":"https://orcid.org/0000-0002-8117-0612"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Teerapong Leelanupab","raw_affiliation_strings":["The University of Queensland, Brisbane, QLD, Australia"],"raw_orcid":"https://orcid.org/0000-0002-8117-0612","affiliations":[{"raw_affiliation_string":"The University of Queensland, Brisbane, QLD, Australia","institution_ids":["https://openalex.org/I165143802"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I165143802"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4322","last_page":"4327"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6413999795913696},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.310699999332428},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.28850001096725464},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.28110000491142273},{"id":"https://openalex.org/keywords/intervention","display_name":"Intervention (counseling)","score":0.2705000042915344}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6413999795913696},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.6247000098228455},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.3691999912261963},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.3334999978542328},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.310699999332428},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.28850001096725464},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.28110000491142273},{"id":"https://openalex.org/C2780665704","wikidata":"https://www.wikidata.org/wiki/Q959298","display_name":"Intervention (counseling)","level":2,"score":0.2705000042915344},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.25920000672340393},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.2515999972820282}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805712.3809905","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3809905","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805712.3809905","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3809905","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W1965690069","https://openalex.org/W2740069864","https://openalex.org/W4385688511","https://openalex.org/W4387561528","https://openalex.org/W4399554477","https://openalex.org/W4400526908","https://openalex.org/W4401863498","https://openalex.org/W4405143965","https://openalex.org/W4405767463","https://openalex.org/W4407012542","https://openalex.org/W4408811965","https://openalex.org/W4410502462","https://openalex.org/W4412377928","https://openalex.org/W4412396369","https://openalex.org/W4412673545","https://openalex.org/W7133239970","https://openalex.org/W7140193032"],"related_works":[],"abstract_inverted_index":{"Human":[0],"relevance":[1,40,52,66,87,118,145,161],"assessment":[2],"is":[3],"time-consuming":[4],"and":[5,45,74,76,127,147,165],"cognitively":[6],"intensive,":[7],"limiting":[8],"the":[9,98,136,149],"scalability":[10],"of":[11,61,138],"Information":[12],"Retrieval":[13],"evaluation.":[14],"This":[15],"has":[16],"led":[17],"to":[18,48,124],"growing":[19],"interest":[20],"in":[21,64,110],"using":[22],"large":[23],"language":[24],"models":[25,83],"(LLMs)":[26],"as":[27,140],"proxies":[28],"for":[29,51,143,152,160],"human":[30,144],"judges.":[31],"However,":[32],"it":[33],"remains":[34],"an":[35],"open":[36],"question":[37],"whether":[38],"LLM-based":[39,65,117],"judgments":[41,67,119],"are":[42,167],"reliable,":[43],"stable,":[44],"rigorous":[46],"enough":[47],"match":[49],"humans":[50],"assessment.":[53],"In":[54],"this":[55],"work,":[56],"we":[57],"conduct":[58],"a":[59,103],"study":[60],"overrating":[62],"behavior":[63],"across":[68],"model":[69],"backbones,":[70],"evaluation":[71,155],"paradigms":[72],"(pointwise":[73],"pairwise),":[75],"passage":[77,125],"modification":[78],"strategies.":[79],"We":[80],"show":[81,115],"that":[82,93,116],"consistently":[84],"assign":[85],"inflated":[86],"scores-often":[88],"with":[89],"high":[90],"confidence-to":[91],"passages":[92],"do":[94],"not":[95],"genuinely":[96],"satisfy":[97],"underlying":[99],"information":[100],"need,":[101],"revealing":[102],"system-wide":[104],"bias":[105],"rather":[106],"than":[107],"random":[108],"fluctuations":[109],"judgment.":[111],"Furthermore,":[112],"controlled":[113],"experiments":[114],"can":[120],"be":[121],"highly":[122],"sensitive":[123],"length":[126],"surface-level":[128],"lexical":[129],"cues.":[130],"These":[131],"results":[132,166],"raise":[133],"concerns":[134],"about":[135],"usage":[137],"LLMs":[139,159],"drop-in":[141],"replacements":[142],"assessors,":[146],"highlight":[148],"urgent":[150],"need":[151],"careful":[153],"diagnostic":[154],"frameworks":[156],"when":[157],"applying":[158],"assessments.":[162],"Our":[163],"code":[164],"publicly":[168],"available.":[169],"https://github.com/chutingyu/Exploring-Overrating":[170]},"counts_by_year":[],"updated_date":"2026-07-17T05:52:16.776730","created_date":"2026-07-16T00:00:00"}