{"id":"https://openalex.org/W4412377058","doi":"https://doi.org/10.1145/3726302.3730090","title":"The Great Nugget Recall: Automating Fact Extraction and RAG Evaluation with Large Language Models","display_name":"The Great Nugget Recall: Automating Fact Extraction and RAG Evaluation with Large Language Models","publication_year":2025,"publication_date":"2025-07-13","ids":{"openalex":"https://openalex.org/W4412377058","doi":"https://doi.org/10.1145/3726302.3730090"},"language":"en","primary_location":{"id":"doi:10.1145/3726302.3730090","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730090","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730090","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730090","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101480198","display_name":"Ronak Pradeep","orcid":"https://orcid.org/0000-0001-6296-601X"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Ronak Pradeep","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0000-0001-6296-601X","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052977545","display_name":"Nandan Thakur","orcid":"https://orcid.org/0000-0001-6107-2460"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Nandan Thakur","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0000-0001-6107-2460","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103308060","display_name":"Shivani Upadhyay","orcid":"https://orcid.org/0009-0007-7071-2344"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Shivani Upadhyay","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0009-0007-7071-2344","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103325247","display_name":"Daniel Campos","orcid":"https://orcid.org/0000-0002-5138-8426"},"institutions":[{"id":"https://openalex.org/I142600864","display_name":"College of San Mateo","ror":"https://ror.org/01gwn6z70","country_code":"US","type":"education","lineage":["https://openalex.org/I142600864"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel Campos","raw_affiliation_strings":["Snowflake, San Mateo, USA"],"raw_orcid":"https://orcid.org/0000-0002-5138-8426","affiliations":[{"raw_affiliation_string":"Snowflake, San Mateo, USA","institution_ids":["https://openalex.org/I142600864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055132321","display_name":"Nick Craswell","orcid":"https://orcid.org/0000-0002-9351-8137"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nick Craswell","raw_affiliation_strings":["Microsoft, Seattle, USA"],"raw_orcid":"https://orcid.org/0000-0002-9351-8137","affiliations":[{"raw_affiliation_string":"Microsoft, Seattle, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053747568","display_name":"Ian Soboroff","orcid":"https://orcid.org/0000-0003-2363-3014"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ian Soboroff","raw_affiliation_strings":["NIST, Gaithersburg, USA"],"raw_orcid":"https://orcid.org/0000-0003-2363-3014","affiliations":[{"raw_affiliation_string":"NIST, Gaithersburg, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109409280","display_name":"Hoa Trang Dang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hoa Trang Dang","raw_affiliation_strings":["NIST, Gaithersburg, USA"],"raw_orcid":"https://orcid.org/0009-0006-3009-3465","affiliations":[{"raw_affiliation_string":"NIST, Gaithersburg, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082997975","display_name":"Jimmy Lin","orcid":"https://orcid.org/0000-0002-0661-7189"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jimmy Lin","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0000-0002-0661-7189","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5101480198"],"corresponding_institution_ids":["https://openalex.org/I151746483"],"apc_list":null,"apc_paid":null,"fwci":13.0395,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.98454854,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"180","last_page":"190"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7547250986099243},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.6316668391227722},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.520248532295227},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5000150203704834},{"id":"https://openalex.org/keywords/extraction","display_name":"Extraction (chemistry)","score":0.49872398376464844},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4285638630390167},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3673613667488098},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.1209297776222229}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7547250986099243},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.6316668391227722},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.520248532295227},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5000150203704834},{"id":"https://openalex.org/C4725764","wikidata":"https://www.wikidata.org/wiki/Q844704","display_name":"Extraction (chemistry)","level":2,"score":0.49872398376464844},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4285638630390167},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3673613667488098},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.1209297776222229},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3726302.3730090","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730090","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730090","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3726302.3730090","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730090","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730090","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"},{"id":"https://openalex.org/F4320328359","display_name":"Ministry of Science and ICT, South Korea","ror":"https://ror.org/01wpjm123"},{"id":"https://openalex.org/F4320332178","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"},{"id":"https://openalex.org/F4320335489","display_name":"Institute for Information and Communications Technology Promotion","ror":"https://ror.org/01g0hqq23"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412377058.pdf","grobid_xml":"https://content.openalex.org/works/W4412377058.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W134702066","https://openalex.org/W1974205932","https://openalex.org/W2001575924","https://openalex.org/W2003150093","https://openalex.org/W2015338694","https://openalex.org/W2017633929","https://openalex.org/W2021420765","https://openalex.org/W2075893676","https://openalex.org/W2100520226","https://openalex.org/W2160892561","https://openalex.org/W2560674852","https://openalex.org/W2598654328","https://openalex.org/W2605212342","https://openalex.org/W2606555609","https://openalex.org/W2609145321","https://openalex.org/W3027879771","https://openalex.org/W3156789018","https://openalex.org/W4206396334","https://openalex.org/W4233907442","https://openalex.org/W4243800839","https://openalex.org/W4299527668","https://openalex.org/W4385571412","https://openalex.org/W4389519598","https://openalex.org/W4389520670","https://openalex.org/W4389524022","https://openalex.org/W4393147129","https://openalex.org/W4396812188","https://openalex.org/W4401042808","https://openalex.org/W4401330297","https://openalex.org/W4404780849","https://openalex.org/W4404783449","https://openalex.org/W4411630045","https://openalex.org/W4412230188","https://openalex.org/W6602744131","https://openalex.org/W6629064985"],"related_works":["https://openalex.org/W2118758177","https://openalex.org/W4330338194","https://openalex.org/W2153520307","https://openalex.org/W2151459719","https://openalex.org/W623261610","https://openalex.org/W2316630966","https://openalex.org/W3133744317","https://openalex.org/W4401374939","https://openalex.org/W2036523811","https://openalex.org/W1976696937"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"significantly":[5],"enhanced":[6],"the":[7,19,52,69,100,120,123,165,212],"capabilities":[8],"of":[9,21,122,214],"information":[10],"access":[11],"systems,":[12],"especially":[13],"with":[14],"retrieval-augmented":[15],"generation":[16],"(RAG).":[17],"Nevertheless,":[18],"evaluation":[20,41,54,175,198],"RAG":[22,62,126,216],"systems":[23,78],"remains":[24],"a":[25,30,57,130,157],"barrier":[26],"to":[27,107,116,150,210,223,233],"continued":[28],"progress,":[29],"challenge":[31],"we":[32,98,128,160],"tackle":[33],"in":[34,75,87,228],"this":[35,95],"work":[36],"by":[37,143],"proposing":[38],"an":[39],"automatic":[40,132,173],"framework":[42,102,185,199],"that":[43,51,83,103,196,206],"is":[44,181,221],"validated":[45],"against":[46,134],"human":[47,144],"annotations.":[48],"We":[49],"believe":[50],"nugget":[53,174,189],"methodology":[55],"provides":[56,200],"solid":[58],"foundation":[59],"for":[60,68],"evaluating":[61],"systems.":[63,217],"This":[64,194],"approach,":[65,226],"originally":[66],"developed":[67],"TREC":[70,124],"Question":[71],"Answering":[72],"(QA)":[73],"Track":[74],"2003,":[76],"evaluates":[77],"based":[79],"on":[80,93,154],"atomic":[81],"facts":[82],"should":[84],"be":[85,208],"present":[86],"good":[88],"answers.":[89,118,152],"Our":[90],"efforts":[91],"focus":[92],"''refactoring''":[94],"methodology,":[96],"where":[97,136],"describe":[99],"AutoNuggetizer":[101],"specifically":[104],"applies":[105],"LLMs":[106],"both":[108],"automatically":[109,113],"create":[110],"nuggets":[111,115,137],"and":[112,146,176,204],"assign":[114],"system":[117,151,235],"In":[119],"context":[121],"2024":[125],"Track,":[127],"calibrate":[129],"fully":[131,172],"approach":[133],"strategies":[135],"are":[138,191],"created":[139],"manually":[140,149],"or":[141],"semi-manually":[142],"assessors":[145],"then":[147],"assigned":[148],"Based":[153],"results":[155],"from":[156,171],"community-wide":[158],"evaluation,":[159],"observe":[161],"strong":[162],"agreement":[163,180,232],"at":[164],"run":[166],"level":[167],"between":[168,202],"scores":[169],"derived":[170],"human-based":[177],"variants.":[178],"The":[179],"stronger":[182],"when":[183],"individual":[184],"components":[186],"such":[187],"as":[188],"assignment":[190],"automated":[192],"independently.":[193],"suggests":[195],"our":[197,225],"tradeoffs":[201],"effort":[203],"quality":[205],"can":[207],"used":[209],"guide":[211],"development":[213],"future":[215],"However,":[218],"further":[219],"research":[220],"necessary":[222],"refine":[224],"particularly":[227],"establishing":[229],"robust":[230],"per-topic":[231],"diagnose":[234],"failures":[236],"effectively.":[237]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":3}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}