{"id":"https://openalex.org/W4400525230","doi":"https://doi.org/10.1145/3626772.3657784","title":"Are Large Language Models Good at Utility Judgments?","display_name":"Are Large Language Models Good at Utility Judgments?","publication_year":2024,"publication_date":"2024-07-10","ids":{"openalex":"https://openalex.org/W4400525230","doi":"https://doi.org/10.1145/3626772.3657784"},"language":"en","primary_location":{"id":"doi:10.1145/3626772.3657784","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3626772.3657784","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3626772.3657784","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034939611","display_name":"Hengran Zhang","orcid":"https://orcid.org/0009-0004-1144-1298"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hengran Zhang","raw_affiliation_strings":["CAS Key Lab of Network Data Science and Technology, ICT, CAS & University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0004-1144-1298","affiliations":[{"raw_affiliation_string":"CAS Key Lab of Network Data Science and Technology, ICT, CAS & University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009898523","display_name":"Ruqing Zhang","orcid":"https://orcid.org/0000-0003-4294-2541"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruqing Zhang","raw_affiliation_strings":["CAS Key Lab of Network Data Science and Technology, ICT, CAS & University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-4294-2541","affiliations":[{"raw_affiliation_string":"CAS Key Lab of Network Data Science and Technology, ICT, CAS & University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088621320","display_name":"Jiafeng Guo","orcid":"https://orcid.org/0000-0002-9509-8674"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiafeng Guo","raw_affiliation_strings":["CAS Key Lab of Network Data Science and Technology, ICT, CAS & University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9509-8674","affiliations":[{"raw_affiliation_string":"CAS Key Lab of Network Data Science and Technology, ICT, CAS & University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031439294","display_name":"Maarten de Rijke","orcid":"https://orcid.org/0000-0002-1086-0202"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maarten de Rijke","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-1086-0202","affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006971161","display_name":"Yixing Fan","orcid":"https://orcid.org/0000-0003-4317-2702"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yixing Fan","raw_affiliation_strings":["CAS Key Lab of Network Data Science and Technology, ICT, CAS & University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-4317-2702","affiliations":[{"raw_affiliation_string":"CAS Key Lab of Network Data Science and Technology, ICT, CAS & University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029998682","display_name":"Xueqi Cheng","orcid":"https://orcid.org/0000-0002-5201-8195"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xueqi Cheng","raw_affiliation_strings":["CAS Key Lab of Network Data Science and Technology, ICT, CAS & University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-5201-8195","affiliations":[{"raw_affiliation_string":"CAS Key Lab of Network Data Science and Technology, ICT, CAS & University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.2791,"has_fulltext":false,"cited_by_count":19,"citation_normalized_percentile":{"value":0.96114622,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1941","last_page":"1951"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9656999707221985,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.654169499874115}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.654169499874115}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3626772.3657784","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3626772.3657784","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3626772.3657784","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3626772.3657784","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W1985554184","https://openalex.org/W2101105183","https://openalex.org/W2151012289","https://openalex.org/W2898905149","https://openalex.org/W2911319979","https://openalex.org/W2912924812","https://openalex.org/W3027879771","https://openalex.org/W3155895380","https://openalex.org/W3197202805","https://openalex.org/W4221143046","https://openalex.org/W4229065825","https://openalex.org/W4281557260","https://openalex.org/W4301420909","https://openalex.org/W4382202735","https://openalex.org/W4385570777","https://openalex.org/W4385574269","https://openalex.org/W4387846590","https://openalex.org/W4401043313","https://openalex.org/W6610577358","https://openalex.org/W6838865847"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"Retrieval-augmented":[0],"generation":[1],"(RAG)":[2],"is":[3],"considered":[4],"to":[5,10,30,51,64],"be":[6],"a":[7],"promising":[8],"approach":[9],"alleviate":[11],"the":[12,31,34,40,47,60,66,80],"hallucination":[13],"issue":[14],"of":[15,37,42,49,62,68,82],"large":[16],"language":[17],"models":[18],"(LLMs),":[19],"and":[20],"it":[21],"has":[22,74],"received":[23],"widespread":[24],"attention":[25],"from":[26],"researchers":[27],"recently.":[28],"Due":[29],"limitation":[32],"in":[33,70,84],"semantic":[35],"understanding":[36],"retrieval":[38],"models,":[39],"success":[41],"RAG":[43],"heavily":[44],"lies":[45],"on":[46,78],"ability":[48,61],"LLMs":[50,63],"identify":[52],"passages":[53,69,83],"with":[54],"utility.":[55],"Recent":[56],"efforts":[57],"have":[58],"explored":[59],"assess":[65],"relevance":[67],"retrieval,":[71],"but":[72],"there":[73],"been":[75],"limited":[76],"work":[77],"evaluating":[79],"utility":[81],"supporting":[85],"question":[86],"answering.":[87]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":1}],"updated_date":"2025-12-26T23:08:49.675405","created_date":"2025-10-10T00:00:00"}