{"id":"https://openalex.org/W4391494845","doi":"https://doi.org/10.1162/tacl_a_00632","title":"Benchmarking Large Language Models for News Summarization","display_name":"Benchmarking Large Language Models for News Summarization","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4391494845","doi":"https://doi.org/10.1162/tacl_a_00632"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00632","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00632","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00632/2325685/tacl_a_00632.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00632/2325685/tacl_a_00632.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100437453","display_name":"Tianyi Zhang","orcid":"https://orcid.org/0000-0002-5468-9347"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Tianyi Zhang","raw_affiliation_strings":["Stanford University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074277807","display_name":"Faisal Ladhak","orcid":"https://orcid.org/0009-0009-5730-3866"},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Faisal Ladhak","raw_affiliation_strings":["Columbia University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Columbia University, USA","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001994692","display_name":"Esin Durmus","orcid":"https://orcid.org/0009-0009-7331-8160"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Esin Durmus","raw_affiliation_strings":["Stanford University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025255782","display_name":"Percy Liang","orcid":"https://orcid.org/0000-0002-0458-6139"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Percy Liang","raw_affiliation_strings":["Stanford University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109565051","display_name":"Kathleen McKeown","orcid":null},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kathleen McKeown","raw_affiliation_strings":["Columbia University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Columbia University, USA","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015518638","display_name":"Tatsunori Hashimoto","orcid":"https://orcid.org/0000-0003-0521-5855"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tatsunori B. Hashimoto","raw_affiliation_strings":["Stanford University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University, USA","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100437453"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":92.8693,"has_fulltext":true,"cited_by_count":310,"citation_normalized_percentile":{"value":0.99964149,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"12","issue":null,"first_page":"39","last_page":"57"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.9361323714256287},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8890609741210938},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8659186363220215},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5705626606941223},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.45517733693122864},{"id":"https://openalex.org/keywords/multi-document-summarization","display_name":"Multi-document summarization","score":0.4156262278556824},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.41367098689079285},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3746517300605774}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.9361323714256287},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8890609741210938},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8659186363220215},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5705626606941223},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.45517733693122864},{"id":"https://openalex.org/C134714966","wikidata":"https://www.wikidata.org/wiki/Q6934448","display_name":"Multi-document summarization","level":3,"score":0.4156262278556824},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41367098689079285},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3746517300605774},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/tacl_a_00632","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00632","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00632/2325685/tacl_a_00632.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:a2996e9e8fd5404fababd4eb0d61db38","is_oa":false,"landing_page_url":"https://doaj.org/article/a2996e9e8fd5404fababd4eb0d61db38","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 12 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00632","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00632","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00632/2325685/tacl_a_00632.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.8399999737739563,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320315389","display_name":"Open Philanthropy Project","ror":"https://ror.org/004d1k391"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4391494845.pdf"},"referenced_works_count":88,"referenced_works":["https://openalex.org/W1588495139","https://openalex.org/W1964326564","https://openalex.org/W2012561700","https://openalex.org/W2036484499","https://openalex.org/W2048207804","https://openalex.org/W2055452728","https://openalex.org/W2057820243","https://openalex.org/W2065841724","https://openalex.org/W2066972299","https://openalex.org/W2081174553","https://openalex.org/W2083451366","https://openalex.org/W2095609031","https://openalex.org/W2110693578","https://openalex.org/W2118733980","https://openalex.org/W2130942839","https://openalex.org/W2133182690","https://openalex.org/W2140440594","https://openalex.org/W2606974598","https://openalex.org/W2888482885","https://openalex.org/W2962849707","https://openalex.org/W2962965405","https://openalex.org/W2962985882","https://openalex.org/W2970419734","https://openalex.org/W2971274815","https://openalex.org/W2973379954","https://openalex.org/W2999071598","https://openalex.org/W3022685184","https://openalex.org/W3034383590","https://openalex.org/W3034999214","https://openalex.org/W3035252911","https://openalex.org/W3035725000","https://openalex.org/W3087927280","https://openalex.org/W3098968529","https://openalex.org/W3099766584","https://openalex.org/W3101551503","https://openalex.org/W3103450644","https://openalex.org/W3159259047","https://openalex.org/W3176456866","https://openalex.org/W3195577433","https://openalex.org/W3200980294","https://openalex.org/W3205068155","https://openalex.org/W3211624109","https://openalex.org/W3213990450","https://openalex.org/W4206584443","https://openalex.org/W4210834152","https://openalex.org/W4223908421","https://openalex.org/W4224280227","https://openalex.org/W4224308101","https://openalex.org/W4225934689","https://openalex.org/W4226118367","https://openalex.org/W4226278401","https://openalex.org/W4229005866","https://openalex.org/W4244211701","https://openalex.org/W4285294723","https://openalex.org/W4287674181","https://openalex.org/W4297435087","https://openalex.org/W4378189609","https://openalex.org/W4385571140","https://openalex.org/W4404871502","https://openalex.org/W6629226670","https://openalex.org/W6629790064","https://openalex.org/W6630893282","https://openalex.org/W6632455782","https://openalex.org/W6634065907","https://openalex.org/W6634426243","https://openalex.org/W6677096176","https://openalex.org/W6678262379","https://openalex.org/W6679119442","https://openalex.org/W6679434410","https://openalex.org/W6679436768","https://openalex.org/W6682631176","https://openalex.org/W6691495996","https://openalex.org/W6761205521","https://openalex.org/W6765098219","https://openalex.org/W6767858076","https://openalex.org/W6771915120","https://openalex.org/W6778883912","https://openalex.org/W6782465632","https://openalex.org/W6784437298","https://openalex.org/W6797816506","https://openalex.org/W6800751262","https://openalex.org/W6802669662","https://openalex.org/W6809583738","https://openalex.org/W6810081322","https://openalex.org/W6810738896","https://openalex.org/W6811340617","https://openalex.org/W6843135089","https://openalex.org/W6849590751"],"related_works":["https://openalex.org/W2104677027","https://openalex.org/W3164984162","https://openalex.org/W2902627734","https://openalex.org/W2112885393","https://openalex.org/W1990695371","https://openalex.org/W2173208124","https://openalex.org/W2568827738","https://openalex.org/W2099859325","https://openalex.org/W2365100044","https://openalex.org/W2474342320"],"abstract_inverted_index":{"Abstract":[0],"Large":[1],"language":[2],"models":[3],"(LLMs)":[4],"have":[5,61],"shown":[6],"promise":[7],"for":[8],"automatic":[9],"summarization":[10,56],"but":[11],"the":[12,50,53,101],"reasons":[13],"behind":[14],"their":[15],"successes":[16],"are":[17,110],"poorly":[18],"understood.":[19],"By":[20],"conducting":[21],"a":[22],"human":[23,71,85,117],"evaluation":[24,86],"on":[25,114],"ten":[26],"LLMs":[27],"across":[28],"different":[29],"pretraining":[30],"methods,":[31],"prompts,":[32],"and":[33,73,76],"model":[34,47],"scales,":[35],"we":[36,42,83,90,105],"make":[37],"two":[38],"important":[39],"observations.":[40],"First,":[41],"find":[43,106],"instruction":[44],"tuning,":[45],"not":[46],"size,":[48],"is":[49],"key":[51],"to":[52,68,112],"LLM\u2019s":[54],"zero-shot":[55],"capability.":[57],"Second,":[58],"existing":[59],"studies":[60],"been":[62],"limited":[63],"by":[64],"low-quality":[65],"references,":[66],"leading":[67],"underestimates":[69],"of":[70,103],"performance":[72],"lower":[74],"few-shot":[75],"finetuning":[77],"performance.":[78],"To":[79],"better":[80],"evaluate":[81],"LLMs,":[82],"perform":[84],"over":[87],"high-quality":[88],"summaries":[89,109],"collect":[91],"from":[92],"freelance":[93],"writers.":[94],"Despite":[95],"major":[96],"stylistic":[97],"differences":[98],"such":[99],"as":[100],"amount":[102],"paraphrasing,":[104],"that":[107],"LLM":[108],"judged":[111],"be":[113],"par":[115],"with":[116],"written":[118],"summaries.":[119]},"counts_by_year":[{"year":2026,"cited_by_count":57},{"year":2025,"cited_by_count":163},{"year":2024,"cited_by_count":79},{"year":2023,"cited_by_count":11}],"updated_date":"2026-06-02T09:04:35.204637","created_date":"2025-10-10T00:00:00"}