{"id":"https://openalex.org/W4412673545","doi":"https://doi.org/10.1145/3731120.3744605","title":"A Large-Scale Study of Relevance Assessments with Large Language Models Using UMBRELA","display_name":"A Large-Scale Study of Relevance Assessments with Large Language Models Using UMBRELA","publication_year":2025,"publication_date":"2025-07-18","ids":{"openalex":"https://openalex.org/W4412673545","doi":"https://doi.org/10.1145/3731120.3744605"},"language":"en","primary_location":{"id":"doi:10.1145/3731120.3744605","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3731120.3744605","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103308060","display_name":"Shivani Upadhyay","orcid":"https://orcid.org/0009-0007-7071-2344"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Shivani Upadhyay","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101480198","display_name":"Ronak Pradeep","orcid":"https://orcid.org/0000-0001-6296-601X"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Ronak Pradeep","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052977545","display_name":"Nandan Thakur","orcid":"https://orcid.org/0000-0001-6107-2460"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Nandan Thakur","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103325247","display_name":"Daniel Campos","orcid":"https://orcid.org/0000-0002-5138-8426"},"institutions":[{"id":"https://openalex.org/I142600864","display_name":"College of San Mateo","ror":"https://ror.org/01gwn6z70","country_code":"US","type":"education","lineage":["https://openalex.org/I142600864"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel Campos","raw_affiliation_strings":["Snowflake, San Mateo, USA"],"affiliations":[{"raw_affiliation_string":"Snowflake, San Mateo, USA","institution_ids":["https://openalex.org/I142600864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055132321","display_name":"Nick Craswell","orcid":"https://orcid.org/0000-0002-9351-8137"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nick Craswell","raw_affiliation_strings":["Microsoft, Seattle, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Seattle, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053747568","display_name":"Ian Soboroff","orcid":"https://orcid.org/0000-0003-2363-3014"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ian Soboroff","raw_affiliation_strings":["NIST, Gaithersburg, USA"],"affiliations":[{"raw_affiliation_string":"NIST, Gaithersburg, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082997975","display_name":"Jimmy Lin","orcid":"https://orcid.org/0000-0002-0661-7189"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jimmy Lin","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5103308060"],"corresponding_institution_ids":["https://openalex.org/I151746483"],"apc_list":null,"apc_paid":null,"fwci":12.4244,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.9835024,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"358","last_page":"368"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9954000115394592,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7325040102005005},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6517194509506226},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.6473202705383301},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5192989706993103},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3503267765045166},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.08708155155181885},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.07942035794258118}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7325040102005005},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6517194509506226},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.6473202705383301},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5192989706993103},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3503267765045166},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.08708155155181885},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.07942035794258118},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3731120.3744605","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3731120.3744605","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.49000000953674316,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W134702066","https://openalex.org/W1547627874","https://openalex.org/W2015338694","https://openalex.org/W2036663507","https://openalex.org/W2052989395","https://openalex.org/W2075893676","https://openalex.org/W2141649520","https://openalex.org/W2152314154","https://openalex.org/W2157251260","https://openalex.org/W2160892561","https://openalex.org/W4206396334","https://openalex.org/W4243800839","https://openalex.org/W4384107234","https://openalex.org/W4384662964","https://openalex.org/W4385688511","https://openalex.org/W4400526908","https://openalex.org/W6602744131"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W3204019825"],"abstract_inverted_index":{"There":[0],"is":[1],"substantial":[2],"interest":[3],"in":[4,14,69,85,131,180],"applying":[5],"large":[6],"language":[7],"models":[8],"(LLMs)":[9],"to":[10,58,90,100,161,176],"provide":[11],"relevance":[12,44],"assessments":[13,130],"information":[15],"retrieval":[16],"(IR)":[17],"applications":[18],"from":[19,82],"both":[20],"industry":[21],"and":[22,27,51,99,105,136,146],"academia.":[23],"To":[24],"date,":[25],"researchers":[26],"practitioners":[28],"have":[29],"presented":[30],"several":[31],"studies,":[32],"but":[33],"many":[34],"questions":[35],"remain.":[36],"In":[37],"this":[38],"paper,":[39],"we":[40],"examine":[41],"four":[42],"different":[43,59,97],"assessment":[45],"strategies:":[46],"a":[47,76,139,181],"fully":[48,128,173],"manual":[49,129,174],"process":[50],"three":[52,115],"variants":[53],"that":[54,109,152,168],"rely":[55],"on":[56,75],"LLMs":[57,169],"extents":[60],"using":[61,148],"our":[62,149],"tool":[63],"called":[64],"UMBRELA.":[65],"These":[66],"were":[67],"deployed":[68],"the":[70,96,114,121,142],"TREC":[71],"2024":[72],"RAG":[73],"Track":[74],"diverse":[77],"set":[78],"of":[79,133],"77":[80],"runs":[81],"19":[83],"teams":[84],"situ,":[86],"which":[87],"allowed":[88],"us":[89],"correlate":[91,118],"system":[92,110],"rankings":[93,111],"induced":[94],"by":[95,113,127],"approaches":[98],"characterize":[101],"tradeoffs":[102],"between":[103],"cost":[104],"quality.":[106],"We":[107],"find":[108],"produced":[112,126],"LLM-based":[116],"strategies":[117],"well":[119],"at":[120],"run":[122],"level":[123],"with":[124],"those":[125],"terms":[132],"nDCG@20,":[134],"nDCG@100,":[135],"Recall@100.":[137],"On":[138],"topic-by-topic":[140],"basis,":[141],"correlations":[143,159],"are":[144],"lower,":[145],"results":[147],"setup":[150],"indicate":[151],"increased":[153],"human":[154],"involvement":[155],"does":[156],"not":[157],"improve":[158],"sufficiently":[160],"justify":[162],"their":[163],"costs.":[164],"Our":[165],"study":[166],"suggests":[167],"can":[170],"potentially":[171],"replace":[172],"judgments":[175],"measure":[177],"run-level":[178],"effectiveness":[179],"coarse-grained":[182],"manner.":[183]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}