{"id":"https://openalex.org/W2964016842","doi":"https://doi.org/10.63317/3uxkgihvmbtw","title":"Revisiting Summarization Evaluation for Scientific Articles","display_name":"Revisiting Summarization Evaluation for Scientific Articles","alternate_titles":[],"publication_year":2016,"publication_date":"2016-05-01","ids":{"openalex":"https://openalex.org/W2964016842","doi":"https://doi.org/10.63317/3uxkgihvmbtw","mag":"2964016842"},"language":"en","primary_location":{"id":"doi:10.63317/3uxkgihvmbtw","doi":"https://doi.org/10.63317/3uxkgihvmbtw","is_oa":true,"landing_page_url":"https://doi.org/10.63317/3uxkgihvmbtw","pdf_url":"http://www.lrec-conf.org/proceedings/lrec2016/pdf/1144_Paper.pdf","source":{"id":"https://openalex.org/S7407088301","display_name":"Language Resources and Evaluation Conference (LREC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Language Resources and Evaluation Conference","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W2964016842"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"http://www.lrec-conf.org/proceedings/lrec2016/pdf/1144_Paper.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064858748","display_name":"Arman Cohan","orcid":"https://orcid.org/0000-0002-8954-2724","observed_orcids":["https://orcid.org/0000-0002-8954-2724"]},"institutions":[{"id":"https://openalex.org/I184565670","display_name":"Georgetown University","ror":"https://ror.org/05vzafd60","country_code":"US","type":"education","lineage":["https://openalex.org/I184565670"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Arman Cohan","raw_affiliation_strings":["Information Retrieval Lab Department of Computer Science Georgetown University","Georgetown University, Washington, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Information Retrieval Lab Department of Computer Science Georgetown University","institution_ids":["https://openalex.org/I184565670"]},{"raw_affiliation_string":"Georgetown University, Washington, United States","institution_ids":["https://openalex.org/I184565670"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036610566","display_name":"Nazli Goharian","orcid":"https://orcid.org/0000-0002-6349-5237","observed_orcids":["https://orcid.org/0000-0002-6349-5237"]},"institutions":[{"id":"https://openalex.org/I184565670","display_name":"Georgetown University","ror":"https://ror.org/05vzafd60","country_code":"US","type":"education","lineage":["https://openalex.org/I184565670"]},{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Nazli Goharian","raw_affiliation_strings":["Information Retrieval Lab Department of Computer Science Georgetown University","University of Washington, Seattle, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Information Retrieval Lab Department of Computer Science Georgetown University","institution_ids":["https://openalex.org/I184565670"]},{"raw_affiliation_string":"University of Washington, Seattle, United States","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.2969,"has_fulltext":true,"cited_by_count":104,"citation_normalized_percentile":{"value":0.92006555,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"806","last_page":"813"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0008999999845400453,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/3310","display_name":"Linguistics and Language"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/summarization-evaluation","display_name":"summarization evaluation","score":0.8569999933242798},{"id":"https://openalex.org/keywords/rouge","display_name":"ROUGE","score":0.9559999704360962},{"id":"https://openalex.org/keywords/content-relevance","display_name":"content relevance","score":0.6079999804496765},{"id":"https://openalex.org/keywords/lexical-overlap","display_name":"lexical overlap","score":0.5410000085830688},{"id":"https://openalex.org/keywords/paraphrasing","display_name":"paraphrase","score":0.7089999914169312},{"id":"https://openalex.org/keywords/terminology-variation","display_name":"terminology variation","score":0.7749999761581421}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.9799100160598755},{"id":"https://openalex.org/C7888048","wikidata":"https://www.wikidata.org/wiki/Q7277427","display_name":"ROUGE","level":2,"score":0.777141809463501},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7659486532211304},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7414517402648926},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7357701063156128},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6293101906776428},{"id":"https://openalex.org/C547195049","wikidata":"https://www.wikidata.org/wiki/Q1725664","display_name":"Terminology","level":2,"score":0.5700796246528625},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5581033229827881},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4612490236759186},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.42005470395088196},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.41246843338012695},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.19623255729675293},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14403042197227478},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.63317/3uxkgihvmbtw","doi":"https://doi.org/10.63317/3uxkgihvmbtw","is_oa":true,"landing_page_url":"https://doi.org/10.63317/3uxkgihvmbtw","pdf_url":"http://www.lrec-conf.org/proceedings/lrec2016/pdf/1144_Paper.pdf","source":{"id":"https://openalex.org/S7407088301","display_name":"Language Resources and Evaluation Conference (LREC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Language Resources and Evaluation Conference","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1604.00400","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1604.00400","pdf_url":"https://arxiv.org/pdf/1604.00400","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.1604.00400","doi":"https://doi.org/10.48550/arxiv.1604.00400","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1604.00400","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:2964016842","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1604.00400v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"doi:10.63317/3uxkgihvmbtw","doi":"https://doi.org/10.63317/3uxkgihvmbtw","is_oa":true,"landing_page_url":"https://doi.org/10.63317/3uxkgihvmbtw","pdf_url":"http://www.lrec-conf.org/proceedings/lrec2016/pdf/1144_Paper.pdf","source":{"id":"https://openalex.org/S7407088301","display_name":"Language Resources and Evaluation Conference (LREC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Language Resources and Evaluation Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G8457822286","display_name":"II-NEW: Infrastructure for Change: From a Teaching Department to National Prominence","funder_award_id":"1204347","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2964016842.pdf","grobid_xml":"https://content.openalex.org/works/W2964016842.grobid-xml"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W200488758","https://openalex.org/W1520857482","https://openalex.org/W1530435204","https://openalex.org/W1662133657","https://openalex.org/W1880262756","https://openalex.org/W1974339500","https://openalex.org/W1975579663","https://openalex.org/W1983018376","https://openalex.org/W1989420837","https://openalex.org/W2030265142","https://openalex.org/W2083305840","https://openalex.org/W2101105183","https://openalex.org/W2102065370","https://openalex.org/W2110693578","https://openalex.org/W2115637926","https://openalex.org/W2123946480","https://openalex.org/W2136082655","https://openalex.org/W2136542423","https://openalex.org/W2146593092","https://openalex.org/W2147152072","https://openalex.org/W2154652894","https://openalex.org/W2157979304","https://openalex.org/W2251023345","https://openalex.org/W2251670640","https://openalex.org/W2442495973","https://openalex.org/W2914777512","https://openalex.org/W3138773240","https://openalex.org/W4240913316","https://openalex.org/W7099884161"],"related_works":["https://openalex.org/W2251023345","https://openalex.org/W2050715302","https://openalex.org/W2251670640","https://openalex.org/W4410119193","https://openalex.org/W4408258410","https://openalex.org/W2151695628","https://openalex.org/W2890013547","https://openalex.org/W2093969193","https://openalex.org/W7098718453","https://openalex.org/W2135303634"],"abstract_inverted_index":{"Evaluation":[0,158],"of":[1,14,21,54,83,115,177],"text":[2],"summarization":[3,31,65,76,133],"approaches":[4],"have":[5],"been":[6,34],"mostly":[7],"based":[8,137],"on":[9,40,138],"metrics":[10],"that":[11,70],"measure":[12],"similarities":[13],"system":[15,144],"generated":[16,145],"summaries":[17],"with":[18,122,168],"a":[19,143],"set":[20],"human":[22,150],"written":[23,151],"gold-standard":[24],"summaries.The":[25],"most":[26],"widely":[27],"used":[28],"metric":[29,89,131,155],"in":[30,48,52,106,118,175],"evaluation":[32,88,134,176],"has":[33],"the":[35,44,49,98,123,139,148],"ROUGE":[36,59,101,116],"family.ROUGE":[37],"solely":[38],"relies":[39],"lexical":[41],"overlaps":[42],"between":[43,142],"terms":[45],"and":[46,57,147],"phrases":[47],"sentences;":[50],"therefore,":[51],"cases":[53],"terminology":[55],"variations":[56],"paraphrasing,":[58],"is":[60,66,71,102,136],"not":[61,103],"as":[62,86],"effective.Scientific":[63],"article":[64,179],"one":[67],"such":[68],"case":[69],"different":[72,113,120],"from":[73],"general":[74],"domain":[75],"(e.g.newswire":[77],"data).We":[78],"provide":[79],"an":[80,87,129],"extensive":[81],"analysis":[82],"ROUGE's":[84],"effectiveness":[85,174],"for":[90,132],"scientific":[91,108,178],"summarization;":[92],"we":[93,127],"show":[94,111],"that,":[95],"contrary":[96],"to":[97],"common":[99],"belief,":[100],"much":[104],"reliable":[105],"evaluating":[107],"summaries.We":[109,152],"furthermore":[110],"how":[112],"variants":[114],"result":[117],"very":[119],"correlations":[121,167],"manual":[124,169],"Pyramid":[125],"scores.Finally,":[126],"propose":[128],"alternative":[130],"which":[135,171],"content":[140],"relevance":[141],"summary":[146],"corresponding":[149],"call":[153],"our":[154],"SERA":[156,163],"(Summarization":[157],"by":[159],"Relevance":[160],"Analysis).Unlike":[161],"ROUGE,":[162],"consistently":[164],"achieves":[165],"high":[166],"scores":[170],"shows":[172],"its":[173],"summarization.":[180]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2964016842","counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":19},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":12},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":11},{"year":2019,"cited_by_count":10},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":4}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2019-07-30T00:00:00"}
