{"id":"https://openalex.org/W7167913655","doi":"https://doi.org/10.1145/3805712.3809580","title":"Learning to Rank with Multi-Criteria LLM-Judge Annotations","display_name":"Learning to Rank with Multi-Criteria LLM-Judge Annotations","publication_year":2026,"publication_date":"2026-07-10","ids":{"openalex":"https://openalex.org/W7167913655","doi":"https://doi.org/10.1145/3805712.3809580"},"language":null,"primary_location":{"id":"doi:10.1145/3805712.3809580","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3809580","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805712.3809580","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093854193","display_name":"Naghmeh Farzi","orcid":null},"institutions":[{"id":"https://openalex.org/I161057412","display_name":"University of New Hampshire","ror":"https://ror.org/01rmh9n78","country_code":"US","type":"education","lineage":["https://openalex.org/I161057412"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Naghmeh Farzi","raw_affiliation_strings":["University of New Hampshire, Durham, NH, USA"],"raw_orcid":"https://orcid.org/0009-0000-3297-8888","affiliations":[{"raw_affiliation_string":"University of New Hampshire, Durham, NH, USA","institution_ids":["https://openalex.org/I161057412"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027260515","display_name":"Laura Dietz","orcid":"https://orcid.org/0000-0003-1624-3907"},"institutions":[{"id":"https://openalex.org/I161057412","display_name":"University of New Hampshire","ror":"https://ror.org/01rmh9n78","country_code":"US","type":"education","lineage":["https://openalex.org/I161057412"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Laura Dietz","raw_affiliation_strings":["University of New Hampshire, Durham, NH, USA"],"raw_orcid":"https://orcid.org/0000-0003-1624-3907","affiliations":[{"raw_affiliation_string":"University of New Hampshire, Durham, NH, USA","institution_ids":["https://openalex.org/I161057412"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I161057412"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"324","last_page":"335"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2248000055551529,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2248000055551529,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.09380000084638596,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.0568000003695488,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8029000163078308},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6988999843597412},{"id":"https://openalex.org/keywords/complement","display_name":"Complement (music)","score":0.6776999831199646},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.650600016117096},{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"Learning to rank","score":0.636900007724762},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4887999892234802},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.4383000135421753}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8029000163078308},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6988999843597412},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6924999952316284},{"id":"https://openalex.org/C112313634","wikidata":"https://www.wikidata.org/wiki/Q7886648","display_name":"Complement (music)","level":5,"score":0.6776999831199646},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.650600016117096},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.636900007724762},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5738000273704529},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5615000128746033},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4887999892234802},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.4383000135421753},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4284000098705292},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39320001006126404},{"id":"https://openalex.org/C44083865","wikidata":"https://www.wikidata.org/wiki/Q3853443","display_name":"Mean reciprocal rank","level":2,"score":0.3709999918937683},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.32659998536109924},{"id":"https://openalex.org/C161156560","wikidata":"https://www.wikidata.org/wiki/Q1638872","display_name":"Document retrieval","level":2,"score":0.32260000705718994},{"id":"https://openalex.org/C89686163","wikidata":"https://www.wikidata.org/wiki/Q1187982","display_name":"Vector space model","level":2,"score":0.30559998750686646},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.29170000553131104},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2890999913215637},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.288100004196167},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2847000062465668},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.27649998664855957}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805712.3809580","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3809580","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805712.3809580","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3809580","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.6436928510665894,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W1990589796","https://openalex.org/W1997885189","https://openalex.org/W2035720976","https://openalex.org/W2108862644","https://openalex.org/W2955701345","https://openalex.org/W4384107234","https://openalex.org/W4385573057","https://openalex.org/W4385688511","https://openalex.org/W4389518726","https://openalex.org/W4396812188","https://openalex.org/W4400526284","https://openalex.org/W4400527956","https://openalex.org/W4401043313","https://openalex.org/W4401330297","https://openalex.org/W4405767463","https://openalex.org/W4409167096","https://openalex.org/W4412377816","https://openalex.org/W4412377911","https://openalex.org/W4412673258","https://openalex.org/W4412673538","https://openalex.org/W4415316405","https://openalex.org/W7129057328","https://openalex.org/W7140083211","https://openalex.org/W7140214537"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"are":[4],"increasingly":[5],"used":[6],"as":[7,32,39,69],"automated":[8],"judges":[9],"(LLM":[10],"judges)":[11],"to":[12,22,105],"evaluate":[13,77],"Information":[14],"Retrieval":[15],"(IR)":[16],"systems,":[17],"offering":[18],"a":[19,33,40],"cost-effective":[20],"complement":[21],"human":[23],"assessments.":[24],"However,":[25],"most":[26],"prior":[27],"work":[28],"treats":[29],"evaluation":[30],"mainly":[31],"tool":[34],"for":[35,42,73],"comparison":[36],"rather":[37],"than":[38],"signal":[41],"improving":[43],"the":[44,98],"retrieval":[45,106],"system.":[46],"We":[47,76,94],"study":[48],"whether":[49],"criterion":[50],"grades":[51],"from":[52,84],"Multi-Criteria":[53,99],"LLM-Judge":[54,100],"relevance":[55,59,82],"labeling,":[56],"which":[57],"decomposes":[58],"into":[60],"Exactness,":[61],"Coverage,":[62],"Topicality,":[63],"and":[64,91],"Contextual":[65],"Fit,":[66],"can":[67],"serve":[68],"effective":[70],"ranking":[71],"features":[72],"learning-to-rank":[74],"(L2R).":[75],"this":[78],"approach":[79],"using":[80],"manual":[81],"labels":[83],"TREC":[85,86],"DL":[87,89,92],"2019,":[88],"2020,":[90],"2023.":[93],"then":[95],"analyze":[96],"how":[97],"feature":[101],"importance":[102],"varies":[103],"relative":[104],"scores":[107],"across":[108],"IR":[109],"system":[110],"performance":[111],"levels.":[112]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-07-11T00:00:00"}