{"id":"https://openalex.org/W1977019311","doi":"https://doi.org/10.1109/slt.2012.6424269","title":"Ecological validity and the evaluation of speech summarization quality","display_name":"Ecological validity and the evaluation of speech summarization quality","publication_year":2012,"publication_date":"2012-12-01","ids":{"openalex":"https://openalex.org/W1977019311","doi":"https://doi.org/10.1109/slt.2012.6424269","mag":"1977019311"},"language":"en","primary_location":{"id":"doi:10.1109/slt.2012.6424269","is_oa":false,"landing_page_url":"https://doi.org/10.1109/slt.2012.6424269","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2012 IEEE Spoken Language Technology Workshop (SLT)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://nrc-publications.canada.ca/eng/view/accepted/?id=55ccecc7-a877-4f4e-a5bc-da5e66b948c2","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011104235","display_name":"Anthony McCallum","orcid":null},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Anthony McCallum","raw_affiliation_strings":["Department of Computer Science, University of Toronto, Canada","University of Toronto, Toronto, On, Canada"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Toronto, Canada","institution_ids":["https://openalex.org/I185261750"]},{"raw_affiliation_string":"University of Toronto, Toronto, On, Canada","institution_ids":["https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052428595","display_name":"Gerald Penn","orcid":"https://orcid.org/0000-0003-3553-8305"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I197604219","display_name":"National Academies of Sciences, Engineering, and Medicine","ror":"https://ror.org/02eq2w707","country_code":"US","type":"government","lineage":["https://openalex.org/I197604219"]}],"countries":["CA","US"],"is_corresponding":false,"raw_author_name":"Gerald Penn","raw_affiliation_strings":["Department of Computer Science, University of Toronto, Canada","National Research Council Canada, Fredericton, NB, Canada#TAB#"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Toronto, Canada","institution_ids":["https://openalex.org/I185261750"]},{"raw_affiliation_string":"National Research Council Canada, Fredericton, NB, Canada#TAB#","institution_ids":["https://openalex.org/I197604219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038473334","display_name":"Cosmin Munteanu","orcid":"https://orcid.org/0000-0002-0635-9124"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210159778","display_name":"National Research Council Canada","ror":"https://ror.org/04mte1k06","country_code":"CA","type":"government","lineage":["https://openalex.org/I4210159778"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Cosmin Munteanu","raw_affiliation_strings":["National Research Council Canada, Canada","University of Toronto, Toronto, On, Canada"],"affiliations":[{"raw_affiliation_string":"National Research Council Canada, Canada","institution_ids":["https://openalex.org/I4210159778"]},{"raw_affiliation_string":"University of Toronto, Toronto, On, Canada","institution_ids":["https://openalex.org/I185261750"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016892586","display_name":"Xiaodan Zhu","orcid":null},"institutions":[{"id":"https://openalex.org/I197604219","display_name":"National Academies of Sciences, Engineering, and Medicine","ror":"https://ror.org/02eq2w707","country_code":"US","type":"government","lineage":["https://openalex.org/I197604219"]},{"id":"https://openalex.org/I4210159778","display_name":"National Research Council Canada","ror":"https://ror.org/04mte1k06","country_code":"CA","type":"government","lineage":["https://openalex.org/I4210159778"]}],"countries":["CA","US"],"is_corresponding":false,"raw_author_name":"Xiaodan Zhu","raw_affiliation_strings":["National Research Council Canada, Canada","Nat. Res. Council, Canada, Ottawa, ON, Canada"],"affiliations":[{"raw_affiliation_string":"National Research Council Canada, Canada","institution_ids":["https://openalex.org/I4210159778"]},{"raw_affiliation_string":"Nat. Res. Council, Canada, Ottawa, ON, Canada","institution_ids":["https://openalex.org/I197604219"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5011104235"],"corresponding_institution_ids":["https://openalex.org/I185261750"],"apc_list":null,"apc_paid":null,"fwci":0.8918,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.78604352,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":96},"biblio":{"volume":"1","issue":null,"first_page":"467","last_page":"472"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.9477373361587524},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7809722423553467},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5969550013542175},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.5931617021560669},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.5902250409126282},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5477662086486816},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.49084287881851196},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49025729298591614},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.45835983753204346},{"id":"https://openalex.org/keywords/gold-standard","display_name":"Gold standard (test)","score":0.42124325037002563},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.41597649455070496},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3774958848953247},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.1493324637413025},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09245732426643372}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.9477373361587524},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7809722423553467},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5969550013542175},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.5931617021560669},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.5902250409126282},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5477662086486816},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.49084287881851196},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49025729298591614},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.45835983753204346},{"id":"https://openalex.org/C40993552","wikidata":"https://www.wikidata.org/wiki/Q514654","display_name":"Gold standard (test)","level":2,"score":0.42124325037002563},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.41597649455070496},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3774958848953247},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1493324637413025},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09245732426643372},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C204787440","wikidata":"https://www.wikidata.org/wiki/Q188504","display_name":"Alternative medicine","level":2,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1109/slt.2012.6424269","is_oa":false,"landing_page_url":"https://doi.org/10.1109/slt.2012.6424269","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2012 IEEE Spoken Language Technology Workshop (SLT)","raw_type":"proceedings-article"},{"id":"pmh:oai:cisti-icist.nrc-cnrc.ca:cistinparc:21269996","is_oa":false,"landing_page_url":"https://nrc-publications.canada.ca/eng/view/object/?id=e02c6836-327e-4030-8d89-96d3ef98cd39","pdf_url":null,"source":{"id":"https://openalex.org/S7407055245","display_name":"NPARC","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"pmh:oai:cisti-icist.nrc-cnrc.ca:cistinparc:20262886","is_oa":true,"landing_page_url":"https://nrc-publications.canada.ca/eng/view/object/?id=55ccecc7-a877-4f4e-a5bc-da5e66b948c2","pdf_url":"https://nrc-publications.canada.ca/eng/view/accepted/?id=55ccecc7-a877-4f4e-a5bc-da5e66b948c2","source":{"id":"https://openalex.org/S7407055245","display_name":"NPARC","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.361.8308","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.361.8308","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.aclweb.org/anthology/W/W12/W12-2604.pdf","raw_type":"text"},{"id":"mag:1977019311","is_oa":false,"landing_page_url":"https://aclanthology.org/W12-2604/","pdf_url":null,"source":{"id":"https://openalex.org/S4306420633","display_name":"North American Chapter of the Association for Computational Linguistics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":"North American Chapter of the Association for Computational Linguistics","raw_type":null}],"best_oa_location":{"id":"pmh:oai:cisti-icist.nrc-cnrc.ca:cistinparc:20262886","is_oa":true,"landing_page_url":"https://nrc-publications.canada.ca/eng/view/object/?id=55ccecc7-a877-4f4e-a5bc-da5e66b948c2","pdf_url":"https://nrc-publications.canada.ca/eng/view/accepted/?id=55ccecc7-a877-4f4e-a5bc-da5e66b948c2","source":{"id":"https://openalex.org/S7407055245","display_name":"NPARC","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W1977019311.pdf","grobid_xml":"https://content.openalex.org/works/W1977019311.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W1516768428","https://openalex.org/W1753146643","https://openalex.org/W1960178247","https://openalex.org/W1966663330","https://openalex.org/W2039205211","https://openalex.org/W2047865878","https://openalex.org/W2083305840","https://openalex.org/W2085575316","https://openalex.org/W2086044694","https://openalex.org/W2098507392","https://openalex.org/W2115230584","https://openalex.org/W2135303634","https://openalex.org/W2154652894","https://openalex.org/W3138773240","https://openalex.org/W6637625583","https://openalex.org/W6671725215","https://openalex.org/W6677213770","https://openalex.org/W6682631176"],"related_works":["https://openalex.org/W2902865746","https://openalex.org/W2151695628","https://openalex.org/W3040721418","https://openalex.org/W3085939759","https://openalex.org/W11991885","https://openalex.org/W3088169793","https://openalex.org/W2973276266","https://openalex.org/W3214124396","https://openalex.org/W3015517925","https://openalex.org/W2072050228","https://openalex.org/W3201312578","https://openalex.org/W2250352339","https://openalex.org/W1611395667","https://openalex.org/W108011198","https://openalex.org/W2074467802","https://openalex.org/W3119870751","https://openalex.org/W3211402614","https://openalex.org/W3016234399","https://openalex.org/W2805039358","https://openalex.org/W2406273059"],"abstract_inverted_index":{"There":[0],"is":[1,155,193],"little":[2],"evidence":[3],"of":[4,7,37,69,116,136,144,150,188,223],"widespread":[5],"adoption":[6],"speech":[8],"summarization":[9,94],"systems.":[10],"This":[11,59,192],"may":[12,237,257],"be":[13,240],"due":[14],"in":[15,72,76,121,141,172],"part":[16],"to":[17,26,34,97,179,259,276],"the":[18,21,67,74,114,142,147,186,189],"fact":[19],"that":[20,112,140,219],"natural":[22],"language":[23],"heuristics":[24],"used":[25],"generate":[27],"summaries":[28,56,262,279],"are":[29,57,226],"often":[30],"optimized":[31],"with":[32],"respect":[33],"a":[35,70,117,122,128,131,173,201,267],"class":[36],"evaluation":[38,60,99,111,138],"measures":[39,89],"that,":[40],"while":[41],"computationally":[42],"and":[43,90,106,163],"experimentally":[44],"inexpensive,":[45],"rely":[46],"on":[47,185,255],"subjectively":[48],"selected":[49],"gold":[50,132],"standards":[51],"against":[52],"which":[53],"automatically":[54],"generated":[55],"scored.":[58],"protocol":[61],"does":[62],"not":[63,238],"take":[64],"into":[65,273],"account":[66],"usefulness":[68,187,272],"summary":[71,118,129,169,212],"assisting":[73],"listener":[75],"achieving":[77],"his":[78],"or":[79,232],"her":[80],"goal.":[81],"In":[82,214,248],"this":[83],"paper":[84],"we":[85,103],"study":[86],"how":[87,126],"current":[88],"methods":[91],"for":[92,203,245],"evaluating":[93],"systems":[95],"compare":[96],"human-centric":[98],"criteria.":[100],"For":[101],"this,":[102],"have":[104,180],"designed":[105],"conducted":[107],"an":[108],"ecologically":[109,233],"valid":[110,234],"determines":[113],"value":[115],"when":[119,266],"embedded":[120],"task,":[123],"rather":[124],"than":[125,159,166],"closely":[127],"resembles":[130],"standard.":[133],"The":[134],"results":[135,217],"our":[137,216],"demonstrate":[139],"domain":[143],"lecture":[145],"summarization,":[146],"well-known":[148],"baseline":[149],"maximal":[151],"marginal":[152],"relevance":[153],"[1]":[154],"statistically":[156,182],"significantly":[157],"worse":[158,165],"human-generated":[160,261],"extractive":[161],"summaries,":[162,236],"even":[164,265],"having":[167],"no":[168,181,278],"at":[170,280],"all":[171],"simple":[174],"quiz-taking":[175],"task.":[176],"Priming":[177],"seems":[178],"significant":[183],"effect":[184],"human":[190],"summaries.":[191],"interesting":[194],"because":[195],"priming":[196],"had":[197],"been":[198],"proposed":[199],"as":[200,242],"technique":[202],"increasing":[204],"kappa":[205],"scores":[206],"and/or":[207],"maintaining":[208],"goal":[209],"orientation":[210],"among":[211],"authors.":[213],"addition,":[215],"suggest":[218],"ROUGE":[220,256],"scores,":[221],"regardless":[222],"whether":[224],"they":[225],"derived":[227],"from":[228],"numerically-ranked":[229],"reference":[230],"data":[231],"human-extracted":[235],"always":[239],"reliable":[241],"inexpensive":[243],"proxies":[244],"task-embedded":[246,268],"evaluations.":[247],"fact,":[249],"under":[250],"some":[251],"conditions,":[252],"relying":[253],"exclusively":[254],"lead":[258],"scoring":[260],"very":[263],"favourably":[264],"score":[269],"calls":[270],"their":[271],"question":[274],"relative":[275],"using":[277],"all.":[281]},"counts_by_year":[{"year":2014,"cited_by_count":2}],"updated_date":"2026-03-25T23:56:10.502304","created_date":"2025-10-10T00:00:00"}