{"id":"https://openalex.org/W2952245931","doi":"https://doi.org/10.18653/v1/p19-1330","title":"HighRES: Highlight-based Reference-less Evaluation of Summarization","display_name":"HighRES: Highlight-based Reference-less Evaluation of Summarization","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2952245931","doi":"https://doi.org/10.18653/v1/p19-1330","mag":"2952245931"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p19-1330","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1330","pdf_url":"https://www.aclweb.org/anthology/P19-1330.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P19-1330.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049308976","display_name":"Hardy Hardy","orcid":null},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Hardy","raw_affiliation_strings":["Department of Computer Science, University of Sheffield"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Sheffield","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081801388","display_name":"Shashi Narayan","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shashi Narayan","raw_affiliation_strings":["Google Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5067943980","display_name":"Andreas Vlachos","orcid":"https://orcid.org/0000-0003-2123-5071"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]},{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Andreas Vlachos","raw_affiliation_strings":["Department of Computer Science and Technology, University of Cambridge","Department of Computer Science, University of Sheffield"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, University of Cambridge","institution_ids":["https://openalex.org/I241749"]},{"raw_affiliation_string":"Department of Computer Science, University of Sheffield","institution_ids":["https://openalex.org/I91136226"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.0588,"has_fulltext":true,"cited_by_count":36,"citation_normalized_percentile":{"value":0.9387563,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"3381","last_page":"3392"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.9811831712722778},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.876813530921936},{"id":"https://openalex.org/keywords/salient","display_name":"Salient","score":0.6818374395370483},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6717679500579834},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.573733925819397},{"id":"https://openalex.org/keywords/multi-document-summarization","display_name":"Multi-document summarization","score":0.5511823892593384},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.43967074155807495},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3966904282569885},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3739517331123352},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3459767699241638}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.9811831712722778},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.876813530921936},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.6818374395370483},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6717679500579834},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.573733925819397},{"id":"https://openalex.org/C134714966","wikidata":"https://www.wikidata.org/wiki/Q6934448","display_name":"Multi-document summarization","level":3,"score":0.5511823892593384},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43967074155807495},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3966904282569885},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3739517331123352},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3459767699241638},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/p19-1330","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1330","pdf_url":"https://www.aclweb.org/anthology/P19-1330.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/p19-1330","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1330","pdf_url":"https://www.aclweb.org/anthology/P19-1330.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1147893575","display_name":null,"funder_award_id":"EU H2020","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G1901024284","display_name":"eNeMILP: Non-Monotonic Incremental Language Processing","funder_award_id":"EP/R021643/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G495187929","display_name":null,"funder_award_id":"EU\u2010H2020","funder_id":"https://openalex.org/F4320332999","funder_display_name":"Horizon 2020 Framework Programme"},{"id":"https://openalex.org/G5045220161","display_name":null,"funder_award_id":"H2020","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G6243813392","display_name":null,"funder_award_id":"EP/R021643/2","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320328515","display_name":"Lembaga Pengelola Dana Pendidikan","ror":null},{"id":"https://openalex.org/F4320332999","display_name":"Horizon 2020 Framework Programme","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2952245931.pdf","grobid_xml":"https://content.openalex.org/works/W2952245931.grobid-xml"},"referenced_works_count":61,"referenced_works":["https://openalex.org/W1501617060","https://openalex.org/W1520453943","https://openalex.org/W1544827683","https://openalex.org/W1560729591","https://openalex.org/W1975879668","https://openalex.org/W2020861455","https://openalex.org/W2022194976","https://openalex.org/W2026191715","https://openalex.org/W2033442452","https://openalex.org/W2102065370","https://openalex.org/W2120481102","https://openalex.org/W2154652894","https://openalex.org/W2154890045","https://openalex.org/W2158751803","https://openalex.org/W2167007060","https://openalex.org/W2251214593","https://openalex.org/W2270627573","https://openalex.org/W2322584079","https://openalex.org/W2507833193","https://openalex.org/W2512924740","https://openalex.org/W2591784896","https://openalex.org/W2606974598","https://openalex.org/W2727304122","https://openalex.org/W2741375528","https://openalex.org/W2741672218","https://openalex.org/W2760656271","https://openalex.org/W2798542795","https://openalex.org/W2798743277","https://openalex.org/W2799149803","https://openalex.org/W2803583855","https://openalex.org/W2803930360","https://openalex.org/W2804361680","https://openalex.org/W2804624160","https://openalex.org/W2808224048","https://openalex.org/W2808293684","https://openalex.org/W2855233750","https://openalex.org/W2858627132","https://openalex.org/W2888482885","https://openalex.org/W2888912028","https://openalex.org/W2903193068","https://openalex.org/W2962849707","https://openalex.org/W2962884827","https://openalex.org/W2962972512","https://openalex.org/W2962985882","https://openalex.org/W2963227052","https://openalex.org/W2963348916","https://openalex.org/W2963385935","https://openalex.org/W2963537354","https://openalex.org/W2963545005","https://openalex.org/W2963548995","https://openalex.org/W2963607157","https://openalex.org/W2963663567","https://openalex.org/W2963672599","https://openalex.org/W2963926728","https://openalex.org/W2963970666","https://openalex.org/W2964285114","https://openalex.org/W3158986179","https://openalex.org/W4243367687","https://openalex.org/W4250289341","https://openalex.org/W4292356436","https://openalex.org/W4296130438"],"related_works":["https://openalex.org/W3164984162","https://openalex.org/W2104677027","https://openalex.org/W2902627734","https://openalex.org/W2112885393","https://openalex.org/W2173208124","https://openalex.org/W2568827738","https://openalex.org/W1990695371","https://openalex.org/W2365100044","https://openalex.org/W2099859325","https://openalex.org/W2474342320"],"abstract_inverted_index":{"There":[0],"has":[1],"been":[2],"substantial":[3],"progress":[4],"in":[5,63,80,132],"summarization":[6],"research":[7],"enabled":[8],"by":[9,68,90],"the":[10,28,36,38,72,81,87,96,136],"availability":[11],"of":[12,27,60],"novel,":[13],"often":[14],"large-scale,":[15],"datasets":[16],"and":[17,120],"recent":[18],"advances":[19],"on":[20,86],"neural":[21],"network-based":[22],"approaches.":[23,154],"However,":[24],"manual":[25,55],"evaluation":[26,153],"system":[29],"generated":[30],"summaries":[31,65],"is":[32,93],"inconsistent":[33],"due":[34],"to":[35,41,112,134],"difficulty":[37],"task":[39],"poses":[40],"human":[42,91],"non-expert":[43],"readers.":[44],"To":[45,105],"address":[46],"this":[47],"issue,":[48],"we":[49,109],"propose":[50],"a":[51,116],"novel":[52],"approach":[53,108],"for":[54,101],"evaluation,":[56],"HIGHlight-based":[57],"Reference-less":[58],"Evaluation":[59],"Summarization":[61],"(HIGHRES),":[62],"which":[64],"are":[66],"assessed":[67],"multiple":[69,103],"annotators":[70],"against":[71],"source":[73,88,137],"document":[74,89,138],"via":[75],"manually":[76],"highlighted":[77],"salient":[78],"content":[79],"latter.":[82],"Thus":[83],"summary":[84],"assessment":[85],"judges":[92],"facilitated,":[94],"while":[95,140],"highlights":[97,115],"can":[98],"be":[99,149],"used":[100],"evaluating":[102],"systems.":[104,124],"validate":[106],"our":[107],"employ":[110],"crowd-workers":[111],"augment":[113],"with":[114],"recently":[117],"proposed":[118],"dataset":[119],"compare":[121],"two":[122],"state-of-the-art":[123],"We":[125],"demonstrate":[126],"that":[127,147],"HIGHRES":[128],"improves":[129],"inter-annotator":[130],"agreement":[131],"comparison":[133],"using":[135],"directly,":[139],"they":[141],"help":[142],"emphasize":[143],"differences":[144],"among":[145],"systems":[146],"would":[148],"ignored":[150],"under":[151],"other":[152],"1":[155]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":10},{"year":2020,"cited_by_count":8},{"year":2019,"cited_by_count":2}],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2025-10-10T00:00:00"}