{"id":"https://openalex.org/W4400530529","doi":"https://doi.org/10.1145/3626772.3661346","title":"A Field Guide to Automatic Evaluation of LLM-Generated Summaries","display_name":"A Field Guide to Automatic Evaluation of LLM-Generated Summaries","publication_year":2024,"publication_date":"2024-07-10","ids":{"openalex":"https://openalex.org/W4400530529","doi":"https://doi.org/10.1145/3626772.3661346"},"language":"en","primary_location":{"id":"doi:10.1145/3626772.3661346","doi":"https://doi.org/10.1145/3626772.3661346","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3626772.3661346","pdf_url":null,"source":{"id":"https://openalex.org/S4363608773","display_name":"International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3626772.3661346","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071729991","display_name":"Tempest A van Schaik","orcid":"https://orcid.org/0000-0001-7745-2249","observed_orcids":["https://orcid.org/0000-0001-7745-2249"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tempest A. van Schaik","raw_affiliation_strings":["Microsoft, Redmond, USA"],"raw_orcid":"https://orcid.org/0000-0001-7745-2249","affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":null,"display_name":"Brittany Pugh","orcid":"https://orcid.org/0009-0002-9143-632X","observed_orcids":["https://orcid.org/0009-0002-9143-632X"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Brittany Pugh","raw_affiliation_strings":["Microsoft, Redmond, USA"],"raw_orcid":"https://orcid.org/0009-0002-9143-632X","affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":6.3145,"has_fulltext":false,"cited_by_count":40,"citation_normalized_percentile":{"value":0.97464764,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"2832","last_page":"2836"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8637999892234802,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8637999892234802,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.05920000001788139,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.05000000074505806,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-generated-summaries","display_name":"LLM-generated summaries","score":0.7559999823570251},{"id":"https://openalex.org/keywords/automated-assessment","display_name":"automated assessment","score":0.7829999923706055},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8920000195503235},{"id":"https://openalex.org/keywords/text-summarization","display_name":"text summarization","score":0.9150000214576721},{"id":"https://openalex.org/keywords/hallucinations","display_name":"hallucinations","score":0.765999972820282},{"id":"https://openalex.org/keywords/system-level-evaluation","display_name":"system-level evaluation","score":0.7839999794960022},{"id":"https://openalex.org/keywords/evaluation-pitfalls","display_name":"evaluation pitfalls","score":0.45899999141693115}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6816741824150085},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.6124576926231384},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.45397356152534485},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09195095300674438},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3626772.3661346","doi":"https://doi.org/10.1145/3626772.3661346","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3626772.3661346","pdf_url":null,"source":{"id":"https://openalex.org/S4363608773","display_name":"International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3626772.3661346","doi":"https://doi.org/10.1145/3626772.3661346","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3626772.3661346","pdf_url":null,"source":{"id":"https://openalex.org/S4363608773","display_name":"International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W2000777050","https://openalex.org/W2154652894","https://openalex.org/W2936695845","https://openalex.org/W2969286400","https://openalex.org/W2970785793","https://openalex.org/W2970892365","https://openalex.org/W2981456979","https://openalex.org/W3008926737","https://openalex.org/W3027879771","https://openalex.org/W3035628162","https://openalex.org/W3046375318","https://openalex.org/W3093429745","https://openalex.org/W3136687158","https://openalex.org/W3157230343","https://openalex.org/W3211912523","https://openalex.org/W3213990450","https://openalex.org/W4226118367","https://openalex.org/W4283026156","https://openalex.org/W4320003957","https://openalex.org/W4362515116","https://openalex.org/W4378189609","https://openalex.org/W4378465287","https://openalex.org/W4384625752","https://openalex.org/W4385572061","https://openalex.org/W4385573386","https://openalex.org/W4385718146","https://openalex.org/W4385774833","https://openalex.org/W4389519254","https://openalex.org/W4389520163","https://openalex.org/W4389524085","https://openalex.org/W4391591738","https://openalex.org/W4399528455","https://openalex.org/W4402671800","https://openalex.org/W7133218729","https://openalex.org/W7133239970"],"related_works":["https://openalex.org/W7167838648","https://openalex.org/W6929141969","https://openalex.org/W4405955329","https://openalex.org/W7158208954","https://openalex.org/W4387428151","https://openalex.org/W4386875790","https://openalex.org/W7010986548","https://openalex.org/W4389520163","https://openalex.org/W4400343267","https://openalex.org/W6943917543"],"abstract_inverted_index":{"Large":[0],"Language":[1,73],"models":[2],"(LLMs)":[3],"are":[4,53,85],"rapidly":[5],"being":[6],"adopted":[7],"for":[8,25,70,88,175,187],"tasks":[9,76],"such":[10,127],"as":[11,128],"text":[12,117,158],"summarization,":[13],"in":[14,110,147],"a":[15,166],"wide":[16],"range":[17],"of":[18,36,60,80,93,114,123,131,142,156],"industries.":[19],"This":[20],"has":[21],"driven":[22],"the":[23,34,78,129,139,148,157],"need":[24,133],"scalable,":[26],"automatic,":[27],"reliable,":[28],"and":[29,51,168,171],"cost-effective":[30],"methods":[31,66,97,178],"to":[32,101,105,134,179],"evaluate":[33,102],"quality":[35],"LLM-generated":[37,163],"text.":[38],"What":[39],"is":[40,46,165],"meant":[41],"by":[42,160],"evaluating":[43,188],"an":[44,121],"LLM":[45,115,143,189],"not":[47,86,120],"yet":[48],"well":[49],"defined":[50],"there":[52],"widely":[54],"different":[55,140],"expectations":[56],"about":[57],"what":[58],"kind":[59],"information":[61],"evaluation":[62,96,144,155,177],"will":[63],"produce.":[64],"Evaluation":[65],"that":[67,98,118,196],"were":[68,119],"developed":[69],"traditional":[71],"Natural":[72],"Processing":[74],"(NLP)":[75],"(before":[77],"rise":[79],"LLMs)":[81],"remain":[82],"applicable":[83],"but":[84,108,150],"sufficient":[87],"capturing":[89],"high-level":[90],"semantic":[91],"qualities":[92],"summaries.":[94],"Emerging":[95],"use":[99],"LLMs":[100],"LLM-output,":[103],"appear":[104],"be":[106,135],"powerful":[107],"lacking":[109],"reliability.":[111],"New":[112],"elements":[113],"generated":[116,159],"element":[122],"previous":[124],"NLP":[125],"tasks,":[126],"artifacts":[130],"hallucination,":[132],"considered.":[136],"We":[137],"outline":[138],"types":[141],"currently":[145],"used":[146],"literature":[149],"focus":[151],"on":[152],"offline,":[153],"system-level":[154],"LLMs.":[161],"Evaluating":[162],"summaries":[164],"complex":[167],"fast-evolving":[169],"area,":[170],"we":[172,191],"propose":[173],"strategies":[174,186],"applying":[176],"avoid":[180],"common":[181],"pitfalls.":[182],"Despite":[183],"having":[184],"promising":[185],"summaries,":[190],"highlight":[192],"some":[193],"open":[194],"challenges":[195],"remain.":[197]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":27},{"year":2024,"cited_by_count":4}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-07-12T00:00:00"}