{"id":"https://openalex.org/W2129006464","doi":"https://doi.org/10.3115/v1/w14-1809","title":"Automatic evaluation of spoken summaries: the case of language assessment","display_name":"Automatic evaluation of spoken summaries: the case of language assessment","publication_year":2014,"publication_date":"2014-01-01","ids":{"openalex":"https://openalex.org/W2129006464","doi":"https://doi.org/10.3115/v1/w14-1809","mag":"2129006464"},"language":"en","primary_location":{"id":"doi:10.3115/v1/w14-1809","is_oa":true,"landing_page_url":"https://doi.org/10.3115/v1/w14-1809","pdf_url":"https://doi.org/10.3115/v1/w14-1809","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Ninth Workshop on Innovative Use of NLP for Building Educational Applications","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.3115/v1/w14-1809","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064750543","display_name":"Anastassia Loukina","orcid":"https://orcid.org/0000-0002-8213-3362"},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anastassia Loukina","raw_affiliation_strings":["Educational Testing Service"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service","institution_ids":["https://openalex.org/I1341030882"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069537054","display_name":"Klaus Zechner","orcid":"https://orcid.org/0000-0002-9465-6929"},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Klaus Zechner","raw_affiliation_strings":["Educational Testing Service"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service","institution_ids":["https://openalex.org/I1341030882"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100333391","display_name":"Lei Chen","orcid":"https://orcid.org/0000-0001-5263-5540"},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lei Chen","raw_affiliation_strings":["Educational Testing Service"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service","institution_ids":["https://openalex.org/I1341030882"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1341030882"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"68","last_page":"78"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.8231056928634644},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7607291340827942},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.7186782360076904},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6947961449623108},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.6124803423881531},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.6106339693069458},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5474140048027039},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.505821943283081},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.4867110550403595},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4830530285835266},{"id":"https://openalex.org/keywords/spoken-language","display_name":"Spoken language","score":0.4739641845226288},{"id":"https://openalex.org/keywords/precision-and-recall","display_name":"Precision and recall","score":0.4533332288265228},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.14728760719299316},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08847606182098389}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.8231056928634644},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7607291340827942},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7186782360076904},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6947961449623108},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.6124803423881531},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.6106339693069458},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5474140048027039},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.505821943283081},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.4867110550403595},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4830530285835266},{"id":"https://openalex.org/C2776230583","wikidata":"https://www.wikidata.org/wiki/Q1322198","display_name":"Spoken language","level":2,"score":0.4739641845226288},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.4533332288265228},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.14728760719299316},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08847606182098389},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3115/v1/w14-1809","is_oa":true,"landing_page_url":"https://doi.org/10.3115/v1/w14-1809","pdf_url":"https://doi.org/10.3115/v1/w14-1809","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Ninth Workshop on Innovative Use of NLP for Building Educational Applications","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.653.3603","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.653.3603","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://mkzechner.net/bea2014summarization.pdf","raw_type":"text"}],"best_oa_location":{"id":"doi:10.3115/v1/w14-1809","is_oa":true,"landing_page_url":"https://doi.org/10.3115/v1/w14-1809","pdf_url":"https://doi.org/10.3115/v1/w14-1809","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Ninth Workshop on Innovative Use of NLP for Building Educational Applications","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7900000214576721,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W322826253","https://openalex.org/W429766147","https://openalex.org/W1485919316","https://openalex.org/W1520857482","https://openalex.org/W1966359789","https://openalex.org/W1979364015","https://openalex.org/W2025719462","https://openalex.org/W2030026461","https://openalex.org/W2059376792","https://openalex.org/W2101105183","https://openalex.org/W2115230584","https://openalex.org/W2120481102","https://openalex.org/W2154652894","https://openalex.org/W2158181145","https://openalex.org/W2161988551","https://openalex.org/W2168881797","https://openalex.org/W2169297889","https://openalex.org/W2170451231","https://openalex.org/W2170779128","https://openalex.org/W2182959134","https://openalex.org/W2250234831","https://openalex.org/W2251456640","https://openalex.org/W2251904119","https://openalex.org/W2293771131","https://openalex.org/W2398210540","https://openalex.org/W2408752745"],"related_works":["https://openalex.org/W2366403280","https://openalex.org/W1495108544","https://openalex.org/W2091301346","https://openalex.org/W3148229873","https://openalex.org/W4389760904","https://openalex.org/W2150160875","https://openalex.org/W4242223894","https://openalex.org/W4306886878","https://openalex.org/W2358294942","https://openalex.org/W4367460280"],"abstract_inverted_index":{"This":[0],"paper":[1],"investigates":[2],"whether":[3],"ROUGE,":[4,33],"a":[5,73,91,133,146],"popular":[6],"metric":[7,173],"for":[8,90,111,117,187,194],"the":[9,19,38,46,49,65,68,112,139,157,171],"evaluation":[10],"of":[11,21,28,40,48,52,61,67,76,83,109,114,132,149,159],"au-tomated":[12,69],"written":[13],"summaries,":[14,122],"can":[15,102],"be":[16,103],"ap-plied":[17],"to":[18,45,145,177],"assessment":[20],"spoken":[22],"sum-maries":[23,161],"produced":[24],"by":[25,105,126,180],"non-native":[26,53],"speakers":[27,54,84],"English.":[29],"We":[30],"demonstrate":[31],"that":[32,156,170],"with":[34,80,98,141],"its":[35],"emphasis":[36],"on":[37,167],"recall":[39],"infor-mation,":[41],"is":[42,174],"particularly":[43],"suited":[44],"as-sessment":[47],"summarization":[50],"quality":[51],"\u2019":[55,85,100,143],"responses.":[56],"A":[57],"stan-dard":[58],"baseline":[59],"implementation":[60],"ROUGE-1":[62],"computed":[63],"over":[64],"output":[66],"speech":[70,182,195],"recognizer":[71],"has":[72],"Spear-man":[74,147],"correlation":[75,148],"\u03c1":[77,150,191],"=":[78,88,151,185,192],"0.55":[79],"experts\u2019":[81],"scores":[82,101,144],"proficiency":[86],"(\u03c1":[87,184],"0.51":[89],"content-vector":[92],"baseline).":[93],"Further":[94],"in-creases":[95],"in":[96],"agreement":[97],"experts":[99,142],"achieved":[104],"using":[106,127],"types":[107],"instead":[108,131],"tokens":[110],"computation":[113],"word":[115],"fre-quencies":[116],"both":[118],"candidate":[119],"and":[120,169],"reference":[121,129,160],"as":[123,125],"well":[124],"multiple":[128],"summaries":[130],"single":[134],"one.":[135],"These":[136],"modifications":[137],"increase":[138],"cor-relation":[140],"0.65.":[152],"Furthermore,":[153],"we":[154],"found":[155],"choice":[158],"does":[162],"not":[163],"have":[164],"any":[165],"impact":[166],"per-formance,":[168],"adjusted":[172],"also":[175],"robust":[176],"errors":[178],"introduced":[179],"auto-mated":[181],"recognition":[183,196],"0.67":[186],"hu-man":[188],"transcriptions":[189],"vs.":[190],"0.65":[193],"output).":[197],"1":[198]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":1},{"year":2015,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}