{"id":"https://openalex.org/W4412377342","doi":"https://doi.org/10.1145/3726302.3730156","title":"Nugget-based Annotation Protocol and Tool For Evaluating Long-form Retrieval-Augmented Generation","display_name":"Nugget-based Annotation Protocol and Tool For Evaluating Long-form Retrieval-Augmented Generation","publication_year":2025,"publication_date":"2025-07-13","ids":{"openalex":"https://openalex.org/W4412377342","doi":"https://doi.org/10.1145/3726302.3730156"},"language":"en","primary_location":{"id":"doi:10.1145/3726302.3730156","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730156","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730156","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730156","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5062016266","display_name":"Eugene Yang","orcid":"https://orcid.org/0000-0002-0051-1535"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eugene Yang","raw_affiliation_strings":["HLTCOE, Johns Hopkins University, Baltimore, Maryland, USA"],"raw_orcid":"https://orcid.org/0000-0002-0051-1535","affiliations":[{"raw_affiliation_string":"HLTCOE, Johns Hopkins University, Baltimore, Maryland, USA","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037074105","display_name":"Dawn Lawrie","orcid":"https://orcid.org/0000-0001-7347-7086"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dawn Lawrie","raw_affiliation_strings":["HLTCOE, Johns Hopkins University, Baltimore, Maryland, USA"],"raw_orcid":"https://orcid.org/0000-0001-7347-7086","affiliations":[{"raw_affiliation_string":"HLTCOE, Johns Hopkins University, Baltimore, Maryland, USA","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109409280","display_name":"Hoa Trang Dang","orcid":"https://orcid.org/0009-0006-3009-3465"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hoa Dang","raw_affiliation_strings":["NIST, Gaithersburg, Maryland, USA"],"raw_orcid":"https://orcid.org/0009-0006-3009-3465","affiliations":[{"raw_affiliation_string":"NIST, Gaithersburg, Maryland, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053747568","display_name":"Ian Soboroff","orcid":"https://orcid.org/0000-0003-2363-3014"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ian Soboroff","raw_affiliation_strings":["NIST, Gaithersburg, Maryland, USA"],"raw_orcid":"https://orcid.org/0000-0003-2363-3014","affiliations":[{"raw_affiliation_string":"NIST, Gaithersburg, Maryland, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036270568","display_name":"James Mayfield","orcid":"https://orcid.org/0000-0003-3866-3013"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James Mayfield","raw_affiliation_strings":["HLTCOE, Johns Hopkins University, Baltimore, Maryland, USA"],"raw_orcid":"https://orcid.org/0000-0003-3866-3013","affiliations":[{"raw_affiliation_string":"HLTCOE, Johns Hopkins University, Baltimore, Maryland, USA","institution_ids":["https://openalex.org/I145311948"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.12004678,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3999","last_page":"4003"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9919000267982483,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8129040002822876},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.7282019853591919},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.6306155920028687},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5527420043945312},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3601604402065277}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8129040002822876},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7282019853591919},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.6306155920028687},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5527420043945312},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3601604402065277},{"id":"https://openalex.org/C204787440","wikidata":"https://www.wikidata.org/wiki/Q188504","display_name":"Alternative medicine","level":2,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3726302.3730156","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730156","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730156","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3726302.3730156","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730156","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730156","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Zero hunger","id":"https://metadata.un.org/sdg/2","score":0.4699999988079071}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412377342.pdf","grobid_xml":"https://content.openalex.org/works/W4412377342.grobid-xml"},"referenced_works_count":11,"referenced_works":["https://openalex.org/W2130736456","https://openalex.org/W2138090731","https://openalex.org/W2528017411","https://openalex.org/W2598654328","https://openalex.org/W4386576859","https://openalex.org/W4389518925","https://openalex.org/W4389519598","https://openalex.org/W4396812188","https://openalex.org/W4401042999","https://openalex.org/W4404792856","https://openalex.org/W6609335943"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2358755282","https://openalex.org/W2625833328"],"abstract_inverted_index":{"Retrieval-augmented":[0],"generation":[1],"(RAG)":[2],"summarizes":[3],"retrieved":[4],"documents":[5],"into":[6],"a":[7],"text":[8,42],"passage":[9],"that":[10],"fulfills":[11],"the":[12,17,25,34,127,132],"information":[13,27,71,79,101],"need":[14,80],"expressed":[15],"by":[16],"user.":[18],"Such":[19],"generated":[20],"responses":[21],"should":[22],"faithfully":[23],"distill":[24],"relevant":[26],"and":[28,44,62,69,74,86,96,118,134],"provide":[29],"sufficient":[30],"attribution":[31],"back":[32],"to":[33,48,81,99,125],"source":[35],"documents.":[36],"Nugget-based":[37],"evaluation":[38],"was":[39],"proposed":[40],"for":[41,102],"summarization":[43],"has":[45,108],"been":[46],"adapted":[47],"evaluate":[49],"RAG":[50,104],"output":[51],"in":[52],"recent":[53],"shared":[54],"tasks":[55],"such":[56,67],"as":[57],"2024":[58],"TREC":[59],"RAG,":[60],"BioGen,":[61],"NeuCLIR":[63],"tracks.":[64],"However,":[65],"annotating":[66],"detailed":[68],"nuanced":[70],"is":[72],"complex":[73],"errorful.":[75],"Multiple":[76],"pieces":[77],"of":[78],"be":[82],"labeled,":[83],"extracted,":[84],"linked,":[85],"cross-referenced.":[87],"In":[88],"this":[89],"work,":[90],"we":[91],"present":[92],"an":[93],"annotation":[94],"protocol":[95],"tool":[97,107],"tailored":[98],"collecting":[100],"evaluating":[103],"systems.":[105],"Our":[106],"four":[109],"steps:":[110],"nugget":[111,113,120],"creation,":[112],"revision,":[114],"document":[115],"support":[116],"assessment,":[117],"finally,":[119],"alignment.":[121],"Each":[122],"step":[123],"aims":[124],"minimize":[126],"annotator's":[128],"cognitive":[129],"load,":[130],"improving":[131],"efficiency":[133],"reliability.":[135]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}