{"id":"https://openalex.org/W4416017405","doi":"https://doi.org/10.1145/3746252.3761158","title":"CLUE: Using Large Language Models for Judging Document Usefulness in Web Search Evaluation","display_name":"CLUE: Using Large Language Models for Judging Document Usefulness in Web Search Evaluation","publication_year":2025,"publication_date":"2025-11-08","ids":{"openalex":"https://openalex.org/W4416017405","doi":"https://doi.org/10.1145/3746252.3761158"},"language":null,"primary_location":{"id":"doi:10.1145/3746252.3761158","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3746252.3761158","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007330219","display_name":"Xingzhu Wang","orcid":"https://orcid.org/0009-0002-9883-0012"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xingzhu Wang","raw_affiliation_strings":["Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0002-9883-0012","affiliations":[{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054416756","display_name":"Erhan Zhang","orcid":"https://orcid.org/0009-0008-2143-2626"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Erhan Zhang","raw_affiliation_strings":["Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0008-2143-2626","affiliations":[{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013838158","display_name":"Yiqun Chen","orcid":"https://orcid.org/0009-0008-6135-2604"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiqun Chen","raw_affiliation_strings":["Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0008-6135-2604","affiliations":[{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120301646","display_name":"Jinghan Xuan","orcid":"https://orcid.org/0009-0006-7753-2592"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jinghan Xuan","raw_affiliation_strings":["School of Statistics, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0006-7753-2592","affiliations":[{"raw_affiliation_string":"School of Statistics, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088015610","display_name":"Yucheng Hou","orcid":"https://orcid.org/0009-0009-1532-743X"},"institutions":[{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yucheng Hou","raw_affiliation_strings":["Baidu Inc., Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-1532-743X","affiliations":[{"raw_affiliation_string":"Baidu Inc., Beijing, China","institution_ids":["https://openalex.org/I98301712"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yitong Xu","orcid":"https://orcid.org/0009-0009-0872-8024"},"institutions":[{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yitong Xu","raw_affiliation_strings":["Baidu Inc., Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-0872-8024","affiliations":[{"raw_affiliation_string":"Baidu Inc., Beijing, China","institution_ids":["https://openalex.org/I98301712"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ying Nie","orcid":"https://orcid.org/0009-0002-9585-8335"},"institutions":[{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ying Nie","raw_affiliation_strings":["Baidu Inc., Beijing, China"],"raw_orcid":"https://orcid.org/0009-0002-9585-8335","affiliations":[{"raw_affiliation_string":"Baidu Inc., Beijing, China","institution_ids":["https://openalex.org/I98301712"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050255638","display_name":"Shuaiqiang Wang","orcid":"https://orcid.org/0000-0002-9212-1947"},"institutions":[{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuaiqiang Wang","raw_affiliation_strings":["Baidu Inc., Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9212-1947","affiliations":[{"raw_affiliation_string":"Baidu Inc., Beijing, China","institution_ids":["https://openalex.org/I98301712"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101771060","display_name":"Dawei Yin","orcid":"https://orcid.org/0000-0002-0684-6205"},"institutions":[{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dawei Yin","raw_affiliation_strings":["Baidu Inc., Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-0684-6205","affiliations":[{"raw_affiliation_string":"Baidu Inc., Beijing, China","institution_ids":["https://openalex.org/I98301712"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5072119199","display_name":"Jiaxin Mao","orcid":"https://orcid.org/0000-0002-9257-5498"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiaxin Mao","raw_affiliation_strings":["Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9257-5498","affiliations":[{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":10,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.3589,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.92134629,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"3133","last_page":"3143"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.8974000215530396,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.8974000215530396,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.0215000007301569,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.015200000256299973,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6509000062942505},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5795999765396118},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.499099999666214},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.4514000117778778},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.40869998931884766},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.376800000667572},{"id":"https://openalex.org/keywords/context-model","display_name":"Context model","score":0.30889999866485596}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8009999990463257},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6509000062942505},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5795999765396118},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5174000263214111},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.499099999666214},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4846000075340271},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.4514000117778778},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.40869998931884766},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40230000019073486},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.376800000667572},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.36890000104904175},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.34060001373291016},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.30889999866485596},{"id":"https://openalex.org/C171686336","wikidata":"https://www.wikidata.org/wiki/Q3532085","display_name":"Topic model","level":2,"score":0.3034000098705292},{"id":"https://openalex.org/C19889080","wikidata":"https://www.wikidata.org/wiki/Q2835852","display_name":"Beam search","level":3,"score":0.30169999599456787},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2937000095844269},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.26089999079704285},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.2590999901294708},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C3017893058","wikidata":"https://www.wikidata.org/wiki/Q999278","display_name":"User satisfaction","level":2,"score":0.2556999921798706},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3746252.3761158","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3746252.3761158","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4549804759","display_name":null,"funder_award_id":"61902209, 62377044","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":36,"referenced_works":["https://openalex.org/W188694065","https://openalex.org/W214995755","https://openalex.org/W1964357740","https://openalex.org/W1997855593","https://openalex.org/W1998381498","https://openalex.org/W2027382829","https://openalex.org/W2034826792","https://openalex.org/W2058473041","https://openalex.org/W2059120814","https://openalex.org/W2074484048","https://openalex.org/W2091265237","https://openalex.org/W2096946253","https://openalex.org/W2116008435","https://openalex.org/W2131490903","https://openalex.org/W2149590690","https://openalex.org/W2151012289","https://openalex.org/W2152693361","https://openalex.org/W2158254843","https://openalex.org/W2163094209","https://openalex.org/W2190044943","https://openalex.org/W2338216121","https://openalex.org/W2740916867","https://openalex.org/W2741619064","https://openalex.org/W2778330162","https://openalex.org/W2952667223","https://openalex.org/W3010990462","https://openalex.org/W3034645139","https://openalex.org/W3043886961","https://openalex.org/W4384107234","https://openalex.org/W4385688511","https://openalex.org/W4391673302","https://openalex.org/W4392846385","https://openalex.org/W4400525230","https://openalex.org/W4400526908","https://openalex.org/W4401043313","https://openalex.org/W4412378004"],"related_works":[],"abstract_inverted_index":{"The":[0],"widely":[1],"adopted":[2],"Cranfield":[3],"paradigm":[4],"fails":[5],"to":[6,12,33,42,106,124],"adequately":[7],"capture":[8],"user":[9,126],"satisfaction":[10,142],"due":[11],"a":[13,50,71],"weak":[14],"relevance-satisfaction":[15],"correlation.":[16],"Additionally,":[17],"constructing":[18],"test":[19],"collections":[20],"incurs":[21],"high":[22],"relevance":[23],"annotation":[24],"costs.":[25],"To":[26],"address":[27],"these":[28],"two":[29],"limitations,":[30],"we":[31,116],"aim":[32],"explore":[34,107],"the":[35,108,118,138,141],"use":[36],"of":[37,110,140],"large":[38],"language":[39],"models":[40],"(LLMs)":[41],"generate":[43],"multilevel":[44,76],"usefulness":[45,77,89,119,134],"labels.":[46],"We":[47,101],"propose":[48],"CLUE,":[49,84],"user-centric":[51],"evaluation":[52],"method":[53],"that":[54,82,131],"explicitly":[55],"incorporates":[56],"users'":[57],"search":[58,93],"context":[59,94],"and":[60,95],"behavior":[61],"information":[62],"into":[63],"LLMs.":[64],"Inspired":[65],"by":[66,122],"ordinal":[67],"regression,":[68],"it":[69],"employs":[70],"cascade":[72],"structure":[73],"tailored":[74],"for":[75],"judgments.":[78],"Our":[79],"study":[80],"shows":[81],"using":[83],"LLMs":[85],"can":[86],"effectively":[87],"assess":[88],"when":[90],"provided":[91],"with":[92],"behavior,":[96],"outperforming":[97],"third-party":[98],"labeling":[99],"methods.":[100],"also":[102],"conduct":[103],"ablation":[104],"studies":[105],"impact":[109],"each":[111],"component":[112],"in":[113],"CLUE.":[114],"Finally,":[115],"utilize":[117],"labels":[120,135],"generated":[121],"CLUE":[123],"predict":[125],"satisfaction.":[127],"Real-world":[128],"experiments":[129],"reveal":[130],"incorporating":[132],"CLUE's":[133],"significantly":[136],"enhances":[137],"performance":[139],"prediction":[143],"model.":[144]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-11-08T00:00:00"}