{"id":"https://openalex.org/W4412377194","doi":"https://doi.org/10.1145/3726302.3729916","title":"LLM-Assisted Relevance Assessments: When Should We Ask LLMs for Help?","display_name":"LLM-Assisted Relevance Assessments: When Should We Ask LLMs for Help?","publication_year":2025,"publication_date":"2025-07-13","ids":{"openalex":"https://openalex.org/W4412377194","doi":"https://doi.org/10.1145/3726302.3729916"},"language":"en","primary_location":{"id":"doi:10.1145/3726302.3729916","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3729916","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3729916","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3729916","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092998360","display_name":"Rikiya Takehi","orcid":"https://orcid.org/0009-0003-6336-8064"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Rikiya Takehi","raw_affiliation_strings":["Waseda University, Tokyo, Japan"],"affiliations":[{"raw_affiliation_string":"Waseda University, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028019543","display_name":"Ellen M. Voorhees","orcid":"https://orcid.org/0000-0002-5658-2308"},"institutions":[{"id":"https://openalex.org/I1321296531","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416","country_code":"US","type":"funder","lineage":["https://openalex.org/I1321296531","https://openalex.org/I1343035065"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ellen M. Voorhees","raw_affiliation_strings":["National Institute of Standards and Technology, Gaithersburg, MD, USA"],"affiliations":[{"raw_affiliation_string":"National Institute of Standards and Technology, Gaithersburg, MD, USA","institution_ids":["https://openalex.org/I1321296531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023595778","display_name":"Tetsuya Sakai","orcid":"https://orcid.org/0000-0002-6720-963X"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Tetsuya Sakai","raw_affiliation_strings":["Waseda University, Tokyo, Japan"],"affiliations":[{"raw_affiliation_string":"Waseda University, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5053747568","display_name":"Ian Soboroff","orcid":"https://orcid.org/0000-0003-2363-3014"},"institutions":[{"id":"https://openalex.org/I1321296531","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416","country_code":"US","type":"funder","lineage":["https://openalex.org/I1321296531","https://openalex.org/I1343035065"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ian Soboroff","raw_affiliation_strings":["National Institute of Standards and Technology, Gaithersburg, MD, USA"],"affiliations":[{"raw_affiliation_string":"National Institute of Standards and Technology, Gaithersburg, MD, USA","institution_ids":["https://openalex.org/I1321296531"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5092998360"],"corresponding_institution_ids":["https://openalex.org/I150744194"],"apc_list":null,"apc_paid":null,"fwci":51.1977,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.99798961,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"95","last_page":"105"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9611999988555908,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9611999988555908,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9129999876022339,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T14330","display_name":"Library Science and Information Systems","score":0.9085999727249146,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ask-price","display_name":"Ask price","score":0.9053844213485718},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7494428157806396},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.49558699131011963},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.24561527371406555},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.20979741215705872},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.06031304597854614}],"concepts":[{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.9053844213485718},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7494428157806396},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49558699131011963},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.24561527371406555},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.20979741215705872},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.06031304597854614},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3726302.3729916","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3729916","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3729916","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3726302.3729916","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3729916","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3729916","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412377194.pdf","grobid_xml":"https://content.openalex.org/works/W4412377194.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W2008285050","https://openalex.org/W2009954908","https://openalex.org/W2013575152","https://openalex.org/W2029075138","https://openalex.org/W2031536661","https://openalex.org/W2032039936","https://openalex.org/W2057495142","https://openalex.org/W2097246133","https://openalex.org/W2160892561","https://openalex.org/W2500119194","https://openalex.org/W2533861448","https://openalex.org/W2598654328","https://openalex.org/W2751013112","https://openalex.org/W2783389297","https://openalex.org/W2895814542","https://openalex.org/W2900833639","https://openalex.org/W3004392977","https://openalex.org/W3101098106","https://openalex.org/W3103991399","https://openalex.org/W4220728222","https://openalex.org/W4284704763","https://openalex.org/W4284713818","https://openalex.org/W4384107234","https://openalex.org/W4385688511","https://openalex.org/W4387323502","https://openalex.org/W4388594564","https://openalex.org/W4401043313","https://openalex.org/W4405143965"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2168627904","https://openalex.org/W2515552481","https://openalex.org/W1570348318","https://openalex.org/W2015444353","https://openalex.org/W3013494979","https://openalex.org/W156769215","https://openalex.org/W4308101915"],"abstract_inverted_index":{"Test":[0],"collections":[1,18,47],"are":[2,95],"information":[3],"retrieval":[4],"tools":[5],"that":[6],"allow":[7],"researchers":[8],"to":[9,60,78,87,110],"quickly":[10],"and":[11,43,98,114],"easily":[12],"evaluate":[13],"ranking":[14],"algorithms.":[15],"While":[16],"test":[17,46],"have":[19,69],"become":[20,49],"an":[21],"integral":[22],"part":[23],"of":[24,29,73],"IR":[25],"research,":[26],"the":[27,53,71],"process":[28],"data":[30],"creation":[31],"involves":[32],"significant":[33],"manual":[34],"annotation":[35],"effort,":[36],"which":[37,57],"often":[38,99],"makes":[39],"it":[40],"very":[41],"expensive":[42],"time-consuming.":[44],"Consequently,":[45],"could":[48],"too":[50,112],"small":[51],"when":[52],"budget":[54],"is":[55,108],"limited,":[56],"may":[58],"lead":[59],"unstable":[61],"evaluations.":[62],"As":[63],"a":[64,103],"cheaper":[65],"alternative,":[66],"recent":[67],"studies":[68],"proposed":[70],"use":[72],"large":[74],"language":[75],"models":[76],"(LLMs)":[77],"completely":[79],"replace":[80],"human":[81,91],"assessors.":[82],"However,":[83],"while":[84],"LLMs":[85,107],"seem":[86],"somewhat":[88],"correlate":[89],"with":[90,106],"judgments,":[92],"their":[93],"predictions":[94],"not":[96,115],"perfect":[97],"show":[100],"bias.":[101],"Thus,":[102],"complete":[104],"replacement":[105],"argued":[109],"be":[111],"risky":[113],"fully":[116],"reliable.":[117]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}