{"id":"https://openalex.org/W4412378004","doi":"https://doi.org/10.1145/3726302.3730223","title":"LLM-Driven Usefulness Labeling for IR Evaluation","display_name":"LLM-Driven Usefulness Labeling for IR Evaluation","publication_year":2025,"publication_date":"2025-07-13","ids":{"openalex":"https://openalex.org/W4412378004","doi":"https://doi.org/10.1145/3726302.3730223"},"language":"en","primary_location":{"id":"doi:10.1145/3726302.3730223","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730223","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730223","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730223","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5118969401","display_name":"Mouly Dewan","orcid":"https://orcid.org/0009-0005-2350-7079"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mouly Dewan","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0009-0005-2350-7079","affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088558868","display_name":"Jiqun Liu","orcid":"https://orcid.org/0000-0003-3643-2182"},"institutions":[{"id":"https://openalex.org/I8692664","display_name":"University of Oklahoma","ror":"https://ror.org/02aqsxs83","country_code":"US","type":"education","lineage":["https://openalex.org/I8692664"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiqun Liu","raw_affiliation_strings":["The University of Oklahoma, Norman, OK, USA"],"raw_orcid":"https://orcid.org/0000-0003-3643-2182","affiliations":[{"raw_affiliation_string":"The University of Oklahoma, Norman, OK, USA","institution_ids":["https://openalex.org/I8692664"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061319881","display_name":"Chirag Shah","orcid":"https://orcid.org/0000-0002-3797-4293"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chirag Shah","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-3797-4293","affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3055","last_page":"3059"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9908999800682068,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6170845627784729}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6170845627784729}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3726302.3730223","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730223","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730223","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3726302.3730223","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730223","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730223","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412378004.pdf","grobid_xml":"https://content.openalex.org/works/W4412378004.grobid-xml"},"referenced_works_count":22,"referenced_works":["https://openalex.org/W1972802558","https://openalex.org/W2057495142","https://openalex.org/W2113640060","https://openalex.org/W2123937625","https://openalex.org/W2158254843","https://openalex.org/W2338216121","https://openalex.org/W2740916867","https://openalex.org/W2741992952","https://openalex.org/W2798863366","https://openalex.org/W3011445502","https://openalex.org/W3125969194","https://openalex.org/W3155638432","https://openalex.org/W4283762145","https://openalex.org/W4385688511","https://openalex.org/W4393335820","https://openalex.org/W4399554477","https://openalex.org/W4400526126","https://openalex.org/W4400526908","https://openalex.org/W4400528870","https://openalex.org/W4403787503","https://openalex.org/W4404407248","https://openalex.org/W4405143965"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"In":[0],"the":[1,20,62,93,111,139,154],"information":[2],"retrieval":[3],"(IR)":[4],"domain,":[5],"evaluation":[6,58],"plays":[7],"a":[8,41,56,96],"crucial":[9,57],"role":[10],"in":[11,91,123,178],"optimizing":[12,175],"search":[13,64,85,113,125,130],"experiences":[14],"and":[15,48,66,79,116,144,174],"supporting":[16],"diverse":[17],"user":[18,84],"intents.In":[19],"recent":[21],"LLM":[22],"era,":[23],"research":[24,98],"has":[25],"been":[26,36],"conducted":[27],"to":[28,152],"automate":[29],"document":[30],"relevance":[31,72,143],"labels.These":[32],"labels":[33,108,169],"have":[34],"traditionally":[35],"assigned":[37],"by":[38,109,170],"crowd-sourced":[39],"workers,":[40],"process":[42],"that":[43,60,100],"is":[44],"both":[45],"time":[46],"consuming":[47],"costly.This":[49],"study":[50,151],"focuses":[51],"on":[52],"LLM-generated":[53,167],"usefulness":[54,94,107,160,168],"labels,":[55],"metric":[59],"considers":[61],"user's":[63],"intents":[65],"task":[67,114],"objectives,":[68],"an":[69,149],"aspect":[70],"where":[71],"falls":[73],"short.Our":[74],"experiment":[75],"utilizes":[76],"task-level,":[77],"query-level,":[78],"document-level":[80],"features":[81],"along":[82,146],"with":[83,129,147],"behavior":[86],"signals,":[87],"which":[88],"are":[89],"essential":[90],"defining":[92],"of":[95],"document.Our":[97],"finds":[99],"(i)":[101],"pre-trained":[102,118],"LLMs":[103,119,136],"can":[104,137],"generate":[105],"moderate":[106],"understanding":[110],"comprehensive":[112],"session,":[115],"(ii)":[117],"perform":[120],"better":[121],"judgment":[122],"short":[124],"sessions":[126],"when":[127],"provided":[128],"session":[131],"contexts.Furthermore,":[132],"we":[133],"investigate":[134],"whether":[135],"capture":[138],"unique":[140],"divergence":[141],"between":[142],"usefulness,":[145],"conducting":[148],"ablation":[150],"identify":[153],"most":[155],"critical":[156,172],"metrics":[157,173],"for":[158,176],"accurate":[159],"label":[161],"generation.In":[162],"conclusion,":[163],"this":[164],"work":[165],"explores":[166],"evaluating":[171],"practicality":[177],"real-world":[179],"settings.":[180]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}