{"id":"https://openalex.org/W7168244240","doi":"https://doi.org/10.1145/3805713.3820429","title":"LLM-Driven Usefulness Judgment for Web Search Evaluation","display_name":"LLM-Driven Usefulness Judgment for Web Search Evaluation","publication_year":2026,"publication_date":"2026-07-14","ids":{"openalex":"https://openalex.org/W7168244240","doi":"https://doi.org/10.1145/3805713.3820429"},"language":null,"primary_location":{"id":"doi:10.1145/3805713.3820429","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805713.3820429","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805713.3820429","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140580546","display_name":"Mouly Dewan","orcid":"https://orcid.org/0009-0005-2350-7079"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mouly Dewan","raw_affiliation_strings":["Information School, University of Washington, Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0009-0005-2350-7079","affiliations":[{"raw_affiliation_string":"Information School, University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140634271","display_name":"Jiqun Liu","orcid":"https://orcid.org/0000-0003-3643-2182"},"institutions":[{"id":"https://openalex.org/I8692664","display_name":"University of Oklahoma","ror":"https://ror.org/02aqsxs83","country_code":"US","type":"education","lineage":["https://openalex.org/I8692664"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiqun Liu","raw_affiliation_strings":["University of Oklahoma, Norman, OK, USA"],"raw_orcid":"https://orcid.org/0000-0003-3643-2182","affiliations":[{"raw_affiliation_string":"University of Oklahoma, Norman, OK, USA","institution_ids":["https://openalex.org/I8692664"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140666939","display_name":"Aditya Gautam","orcid":"https://orcid.org/0009-0001-7324-4079"},"institutions":[{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aditya Gautam","raw_affiliation_strings":["Facebook, Bellevue, WA, USA"],"raw_orcid":"https://orcid.org/0009-0001-7324-4079","affiliations":[{"raw_affiliation_string":"Facebook, Bellevue, WA, USA","institution_ids":["https://openalex.org/I4210108985","https://openalex.org/I4210114444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5140590964","display_name":"Chirag Shah","orcid":"https://orcid.org/0000-0002-3797-4293"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chirag Shah","raw_affiliation_strings":["Information School, University of Washington, Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-3797-4293","affiliations":[{"raw_affiliation_string":"Information School, University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"44","last_page":"55"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.29649999737739563},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.27649998664855957},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.26249998807907104},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.2563000023365021}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6302000284194946},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.40139999985694885},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3977999985218048},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.29649999737739563},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.27649998664855957},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2648000121116638},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.26249998807907104},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.2563000023365021},{"id":"https://openalex.org/C180198813","wikidata":"https://www.wikidata.org/wiki/Q121182","display_name":"Information system","level":2,"score":0.2493000030517578},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2459000051021576}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805713.3820429","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805713.3820429","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805713.3820429","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805713.3820429","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W1972802558","https://openalex.org/W1998381498","https://openalex.org/W2057495142","https://openalex.org/W2074484048","https://openalex.org/W2113640060","https://openalex.org/W2146938270","https://openalex.org/W2158254843","https://openalex.org/W2338216121","https://openalex.org/W2740916867","https://openalex.org/W2741619064","https://openalex.org/W2741992952","https://openalex.org/W2798523458","https://openalex.org/W2997842202","https://openalex.org/W3011445502","https://openalex.org/W3034645139","https://openalex.org/W3155638432","https://openalex.org/W4281653448","https://openalex.org/W4301523337","https://openalex.org/W4385688511","https://openalex.org/W4393335820","https://openalex.org/W4399554477","https://openalex.org/W4400526126","https://openalex.org/W4400526908","https://openalex.org/W4400528870","https://openalex.org/W4404407248","https://openalex.org/W4412378004","https://openalex.org/W4412673258","https://openalex.org/W4412673538","https://openalex.org/W7129057328"],"related_works":[],"abstract_inverted_index":{"Evaluation":[0,95],"is":[1],"fundamental":[2],"to":[3,39,108,176],"optimizing":[4],"search":[5,17,42,111],"experiences":[6],"and":[7,82,105,130,138,150,158,165,200],"supporting":[8],"diverse":[9],"user":[10,84,151],"intents":[11],"in":[12,45],"Information":[13],"Retrieval":[14],"(IR).":[15],"Traditional":[16],"evaluation":[18,55,59,193],"methods":[19,137],"primarily":[20],"rely":[21],"on":[22,64,167],"relevance":[23,36,65,195],"labels,":[24],"which":[25],"assess":[26],"how":[27],"well":[28],"retrieved":[29],"documents":[30],"match":[31],"a":[32,41,53,97,198],"user's":[33],"query.":[34],"However,":[35],"alone":[37],"fails":[38],"capture":[40],"system's":[43],"effectiveness":[44],"helping":[46],"users":[47],"achieve":[48],"their":[49,163],"goals,":[50],"making":[51],"usefulness":[52,77,124,149,182,204],"critical":[54],"criterion.":[56],"Recent":[57],"LLM-enabled":[58],"works":[60],"have":[61],"mostly":[62],"focused":[63],"label":[66,155,183],"generation.":[67],"In":[68],"this":[69,189],"paper,":[70],"we":[71,171],"explore":[72],"an":[73,173],"alternative":[74],"approach:":[75],"LLM-generated":[76],"labels":[78,125],"that":[79,101],"incorporate":[80],"implicit":[81],"explicit":[83],"behavior":[85,112],"signals":[86],"along":[87],"with":[88,126,133,157],"relevance.":[89],"We":[90,143],"introduce":[91],"Task-aware":[92],"Rubric-based":[93],"Usefulness":[94],"(TRUE),":[96],"reproducible":[98,199],"rubric-driven":[99],"framework":[100,202],"leverages":[102],"iterative":[103],"sampling":[104],"Chain-of-Thought":[106],"reasoning":[107],"model":[109,168],"complex":[110],"patterns.":[113],"Our":[114],"study":[115,175],"shows":[116],"that:":[117],"(i)":[118],"pre-trained":[119],"LLMs":[120,132],"can":[121],"generate":[122],"moderate":[123],"rich":[127],"session-level":[128],"context;":[129],"(ii)":[131],"TRUE":[134],"outperform":[135],"state-of-the-art":[136],"our":[139],"systematically":[140],"constructed":[141],"baseline.":[142],"further":[144],"analyze":[145],"the":[146],"relationship":[147],"between":[148],"satisfaction":[152,160],"by":[153,196],"comparing":[154],"generation":[156],"without":[159],"signals,":[161],"quantifying":[162],"correlation":[164],"impact":[166],"behavior.":[169],"Additionally,":[170],"conduct":[172],"ablation":[174],"identify":[177],"key":[178,207],"features":[179],"for":[180,203],"accurate":[181],"generation,":[184],"enabling":[185],"cost-effective":[186],"evaluation.":[187],"Overall,":[188],"work":[190],"advances":[191],"LLM-based":[192],"beyond":[194],"proposing":[197],"scalable":[201],"judgment,":[205],"addressing":[206],"reproducibility":[208],"challenges.":[209]},"counts_by_year":[],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2026-07-15T00:00:00"}