{"id":"https://openalex.org/W2120481102","doi":"https://doi.org/10.1162/coli_a_00123","title":"Automatically Assessing Machine Summary Content Without a Gold Standard","display_name":"Automatically Assessing Machine Summary Content Without a Gold Standard","publication_year":2012,"publication_date":"2012-08-22","ids":{"openalex":"https://openalex.org/W2120481102","doi":"https://doi.org/10.1162/coli_a_00123","mag":"2120481102"},"language":"en","primary_location":{"id":"doi:10.1162/coli_a_00123","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00123","pdf_url":"https://www.mitpressjournals.org/doi/pdf/10.1162/COLI_a_00123","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":false,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://www.mitpressjournals.org/doi/pdf/10.1162/COLI_a_00123","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5112136402","display_name":"Annie Louis","orcid":"https://orcid.org/0000-0003-4226-252X"},"institutions":[{"id":"https://openalex.org/I36788626","display_name":"California University of Pennsylvania","ror":"https://ror.org/01spssf70","country_code":"US","type":"education","lineage":["https://openalex.org/I36788626"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Annie Louis","raw_affiliation_strings":["University of Pennsylvania","#N# * University of Pennsylvania"],"affiliations":[{"raw_affiliation_string":"University of Pennsylvania","institution_ids":["https://openalex.org/I36788626"]},{"raw_affiliation_string":"#N# * University of Pennsylvania","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032571629","display_name":"Ani Nenkova","orcid":"https://orcid.org/0000-0002-5825-7875"},"institutions":[{"id":"https://openalex.org/I36788626","display_name":"California University of Pennsylvania","ror":"https://ror.org/01spssf70","country_code":"US","type":"education","lineage":["https://openalex.org/I36788626"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ani Nenkova","raw_affiliation_strings":["University of Pennsylvania","#N# * University of Pennsylvania"],"affiliations":[{"raw_affiliation_string":"University of Pennsylvania","institution_ids":["https://openalex.org/I36788626"]},{"raw_affiliation_string":"#N# * University of Pennsylvania","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5112136402"],"corresponding_institution_ids":["https://openalex.org/I36788626"],"apc_list":null,"apc_paid":null,"fwci":13.7018,"has_fulltext":true,"cited_by_count":177,"citation_normalized_percentile":{"value":0.98881094,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":100},"biblio":{"volume":"39","issue":"2","first_page":"267","last_page":"300"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8399924039840698},{"id":"https://openalex.org/keywords/gold-standard","display_name":"Gold standard (test)","score":0.7318803668022156},{"id":"https://openalex.org/keywords/replicate","display_name":"Replicate","score":0.626664936542511},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5927911400794983},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.5792063474655151},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.5698301792144775},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5339074730873108},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5197793841362},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4724369943141937},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4606722593307495},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4523882567882538},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.4248379170894623},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.14766547083854675}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8399924039840698},{"id":"https://openalex.org/C40993552","wikidata":"https://www.wikidata.org/wiki/Q514654","display_name":"Gold standard (test)","level":2,"score":0.7318803668022156},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.626664936542511},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5927911400794983},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.5792063474655151},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5698301792144775},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5339074730873108},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5197793841362},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4724369943141937},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4606722593307495},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4523882567882538},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.4248379170894623},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.14766547083854675},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C204787440","wikidata":"https://www.wikidata.org/wiki/Q188504","display_name":"Alternative medicine","level":2,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1162/coli_a_00123","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00123","pdf_url":"https://www.mitpressjournals.org/doi/pdf/10.1162/COLI_a_00123","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":false,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.360.2048","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.360.2048","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://aclweb.org/anthology/J/J13/J13-2002.pdf","raw_type":"text"},{"id":"pmh:oai:doaj.org/article:de1135abef7b4bb9871e39b2616a5d16","is_oa":true,"landing_page_url":"https://doaj.org/article/de1135abef7b4bb9871e39b2616a5d16","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computational Linguistics, Vol 39, Iss 2 (2021)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/coli_a_00123","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00123","pdf_url":"https://www.mitpressjournals.org/doi/pdf/10.1162/COLI_a_00123","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":false,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1171700966","display_name":null,"funder_award_id":"NSF CAREER","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6671297155","display_name":null,"funder_award_id":"CAREER","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2120481102.pdf","grobid_xml":"https://content.openalex.org/works/W2120481102.grobid-xml"},"referenced_works_count":54,"referenced_works":["https://openalex.org/W9655565","https://openalex.org/W42510783","https://openalex.org/W1479700690","https://openalex.org/W1492231956","https://openalex.org/W1495108544","https://openalex.org/W1602831581","https://openalex.org/W1967082914","https://openalex.org/W1970417391","https://openalex.org/W1975579663","https://openalex.org/W1977747299","https://openalex.org/W1983595289","https://openalex.org/W2006902349","https://openalex.org/W2054211469","https://openalex.org/W2062432826","https://openalex.org/W2065883680","https://openalex.org/W2082512208","https://openalex.org/W2083305840","https://openalex.org/W2085116970","https://openalex.org/W2100238596","https://openalex.org/W2102065370","https://openalex.org/W2104184715","https://openalex.org/W2110481933","https://openalex.org/W2122567461","https://openalex.org/W2123946480","https://openalex.org/W2136657878","https://openalex.org/W2137591918","https://openalex.org/W2140440594","https://openalex.org/W2141927646","https://openalex.org/W2147152072","https://openalex.org/W2148374900","https://openalex.org/W2149593800","https://openalex.org/W2150824314","https://openalex.org/W2150869743","https://openalex.org/W2152992673","https://openalex.org/W2154486363","https://openalex.org/W2154652894","https://openalex.org/W2159107349","https://openalex.org/W2162669657","https://openalex.org/W2168881797","https://openalex.org/W2182959134","https://openalex.org/W2250332012","https://openalex.org/W2276787968","https://openalex.org/W2379843655","https://openalex.org/W2398387993","https://openalex.org/W2400500583","https://openalex.org/W2406741635","https://openalex.org/W2463686631","https://openalex.org/W2582743722","https://openalex.org/W2591804103","https://openalex.org/W2895810819","https://openalex.org/W2911640736","https://openalex.org/W2912150279","https://openalex.org/W3138773240","https://openalex.org/W4236654676"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4224009465","https://openalex.org/W2357241418","https://openalex.org/W4286629047","https://openalex.org/W4306321456","https://openalex.org/W4285260836","https://openalex.org/W3046775127","https://openalex.org/W1984733048","https://openalex.org/W2358805260"],"abstract_inverted_index":{"The":[0,71],"most":[1],"widely":[2],"adopted":[3],"approaches":[4],"for":[5,13,68,198],"evaluation":[6,28,53,121],"of":[7,56,82,119,183,193,204,209,215],"summary":[8,16,102,108,128,189],"content":[9,146],"follow":[10],"some":[11],"protocol":[12],"comparing":[14],"a":[15,44,65,132,187],"with":[17,86,103,154,167,206,220],"gold-standard":[18,162],"human":[19,33,112,126,157,168,221],"summaries,":[20,217],"which":[21,110,138],"are":[22,35,58,139],"traditionally":[23],"called":[24],"model":[25,46,84,127,158],"summaries.":[26,89],"This":[27,202],"paradigm":[29],"falls":[30],"short":[31],"when":[32,42,123],"summaries":[34,85,141,197],"not":[36,62],"available":[37,83,130,176],"and":[38,60,100,190],"becomes":[39],"less":[40],"accurate":[41,213],"only":[43,124,173],"single":[45,156],"is":[47,129],"available.":[48],"We":[49,90,115,135],"propose":[50],"three":[51],"novel":[52],"techniques.":[54],"Two":[55],"them":[57],"model-free":[59],"do":[61],"rely":[63],"on":[64],"gold":[66,133],"standard":[67,75],"the":[69,80,94,97,152,155,161,174,181,191,199,207],"assessment.":[70],"third":[72],"technique":[73],"improves":[74],"automatic":[76,149],"evaluations":[77],"by":[78],"expanding":[79],"set":[81],"chosen":[87,105],"system":[88,140,188,196,216],"show":[91],"that":[92],"quantifying":[93],"similarity":[95],"between":[96,186],"source":[98],"text":[99],"its":[101],"appropriately":[104],"measures":[106],"produces":[107,211],"scores":[109],"replicate":[111],"assessments":[113],"accurately.":[114],"also":[116],"explore":[117,180],"ways":[118],"increasing":[120],"quality":[122],"one":[125,175],"as":[131],"standard.":[134],"introduce":[136],"pseudomodels,":[137],"deemed":[142],"to":[143,148,159,164,171],"contain":[144],"good":[145],"according":[147],"evaluation.":[150],"Combining":[151],"pseudomodels":[153],"form":[160],"leads":[163],"higher":[165],"correlations":[166],"judgments":[169],"compared":[170],"using":[172],"model.":[177],"Finally,":[178],"we":[179],"feasibility":[182],"another":[184],"measure\u2014similarity":[185],"pool":[192],"all":[194],"other":[195],"same":[200],"input.":[201],"method":[203],"comparison":[205],"consensus":[208],"systems":[210],"impressively":[212],"rankings":[214,222],"achieving":[218],"correlation":[219],"above":[223],"0.9.":[224]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":26},{"year":2020,"cited_by_count":23},{"year":2019,"cited_by_count":24},{"year":2018,"cited_by_count":13},{"year":2017,"cited_by_count":12},{"year":2016,"cited_by_count":12},{"year":2015,"cited_by_count":10},{"year":2014,"cited_by_count":12},{"year":2013,"cited_by_count":9}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}