{"id":"https://openalex.org/W2507590263","doi":"https://doi.org/10.21437/interspeech.2016-270","title":"An Expectation Maximization Approach to Joint Modeling of Multidimensional Ratings Derived from Multiple Annotators","display_name":"An Expectation Maximization Approach to Joint Modeling of Multidimensional Ratings Derived from Multiple Annotators","publication_year":2016,"publication_date":"2016-08-29","ids":{"openalex":"https://openalex.org/W2507590263","doi":"https://doi.org/10.21437/interspeech.2016-270","mag":"2507590263","pmid":"https://pubmed.ncbi.nlm.nih.gov/41822308"},"language":"en","primary_location":{"id":"doi:10.21437/interspeech.2016-270","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2016-270","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2016","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC12978255/","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072018684","display_name":"Anil Ramakrishna","orcid":"https://orcid.org/0000-0002-7999-0531"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Anil Ramakrishna","raw_affiliation_strings":["Signal Analysis and Interpretation Lab, University of Southern California, Los Angeles, CA, USA"],"affiliations":[{"raw_affiliation_string":"Signal Analysis and Interpretation Lab, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075937808","display_name":"Rahul Gupta","orcid":"https://orcid.org/0000-0002-9277-3718"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rahul Gupta","raw_affiliation_strings":["Signal Analysis and Interpretation Lab, University of Southern California, Los Angeles, CA, USA"],"affiliations":[{"raw_affiliation_string":"Signal Analysis and Interpretation Lab, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083510283","display_name":"Ruth B. Grossman","orcid":null},"institutions":[{"id":"https://openalex.org/I87650153","display_name":"Emerson College","ror":"https://ror.org/011xz5b94","country_code":"US","type":"education","lineage":["https://openalex.org/I87650153"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ruth B. Grossman","raw_affiliation_strings":["Department of Communication Sciences and Disorders, Emerson College, Boston, MA, USA"],"affiliations":[{"raw_affiliation_string":"Department of Communication Sciences and Disorders, Emerson College, Boston, MA, USA","institution_ids":["https://openalex.org/I87650153"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5010028928","display_name":"Shrikanth Narayanan","orcid":"https://orcid.org/0000-0002-1052-6204"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shrikanth S. Narayanan","raw_affiliation_strings":["Signal Analysis and Interpretation Lab, University of Southern California, Los Angeles, CA, USA"],"affiliations":[{"raw_affiliation_string":"Signal Analysis and Interpretation Lab, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5072018684"],"corresponding_institution_ids":["https://openalex.org/I1174212"],"apc_list":null,"apc_paid":null,"fwci":1.1368,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":{"value":0.86053065,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"2016","issue":null,"first_page":"1555","last_page":"1559"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9919000267982483,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6958051323890686},{"id":"https://openalex.org/keywords/joint","display_name":"Joint (building)","score":0.6882249116897583},{"id":"https://openalex.org/keywords/expectation\u2013maximization-algorithm","display_name":"Expectation\u2013maximization algorithm","score":0.557469367980957},{"id":"https://openalex.org/keywords/maximization","display_name":"Maximization","score":0.514404833316803},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49422648549079895},{"id":"https://openalex.org/keywords/data-modeling","display_name":"Data modeling","score":0.4159037470817566},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.3735673427581787},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.331799179315567},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.28134214878082275},{"id":"https://openalex.org/keywords/maximum-likelihood","display_name":"Maximum likelihood","score":0.2739541828632355},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.2219148576259613},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.18192481994628906}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6958051323890686},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.6882249116897583},{"id":"https://openalex.org/C182081679","wikidata":"https://www.wikidata.org/wiki/Q1275153","display_name":"Expectation\u2013maximization algorithm","level":3,"score":0.557469367980957},{"id":"https://openalex.org/C2776330181","wikidata":"https://www.wikidata.org/wiki/Q18358244","display_name":"Maximization","level":2,"score":0.514404833316803},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49422648549079895},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.4159037470817566},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3735673427581787},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.331799179315567},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.28134214878082275},{"id":"https://openalex.org/C49781872","wikidata":"https://www.wikidata.org/wiki/Q1045555","display_name":"Maximum likelihood","level":2,"score":0.2739541828632355},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2219148576259613},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.18192481994628906},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C170154142","wikidata":"https://www.wikidata.org/wiki/Q150737","display_name":"Architectural engineering","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.21437/interspeech.2016-270","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2016-270","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2016","raw_type":"proceedings-article"},{"id":"pmid:41822308","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41822308","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:12978255","is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC12978255/","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Interspeech","raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:pubmedcentral.nih.gov:12978255","is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC12978255/","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Interspeech","raw_type":"Text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2880869243","display_name":null,"funder_award_id":"R01DC012774","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":9,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W1501669607","https://openalex.org/W1989643076","https://openalex.org/W2014399678","https://openalex.org/W2094035326","https://openalex.org/W2117130368","https://openalex.org/W2129479650","https://openalex.org/W2134305421","https://openalex.org/W2163302275"],"related_works":["https://openalex.org/W1996130883","https://openalex.org/W2748574964","https://openalex.org/W2888483922","https://openalex.org/W1514609475","https://openalex.org/W2367747139","https://openalex.org/W2947806671","https://openalex.org/W1976188970","https://openalex.org/W2990541822","https://openalex.org/W1588885902","https://openalex.org/W1965352811"],"abstract_inverted_index":{"Ratings":[0],"from":[1,70,155],"multiple":[2],"human":[3],"annotators":[4,81,139],"are":[5,31,82],"often":[6],"pooled":[7],"in":[8,116,172],"applications":[9],"where":[10],"the":[11,71,76,80,86,123,138,143,156,173,196],"ground":[12,64,87],"truth":[13,65],"is":[14,119,176,199],"hidden.":[15],"Examples":[16],"include":[17],"annotating":[18],"perceived":[19],"emotions":[20],"and":[21,27,38,74,112,158],"assessing":[22],"quality":[23],"metrics":[24],"for":[25],"speech":[26],"image.":[28],"These":[29],"ratings":[30,77,126,154,193],"not":[32],"restricted":[33],"to":[34,51,101,120,131,161,194],"a":[35,61,94,103,149,181,189],"single":[36],"dimension":[37],"can":[39,67,128],"be":[40,68,129],"multidimensional.":[41],"In":[42],"this":[43,117],"paper,":[44],"we":[45],"propose":[46],"an":[47],"Expectation-Maximization":[48],"based":[49],"algorithm":[50],"model":[52,56,92,163],"such":[53],"ratings.":[54],"Our":[55,114],"assumes":[57],"that":[58,66,75],"there":[59],"exists":[60],"latent":[62],"multidimensional":[63,174],"determined":[69],"observation":[72],"features":[73,157],"provided":[78],"by":[79],"noisy":[83],"versions":[84],"of":[85,107,134,178,192],"truth.":[88],"We":[89,146],"test":[90],"our":[91,162],"on":[93,97,137],"study":[95],"conducted":[96],"children":[98],"with":[99],"autism":[100],"predict":[102,122,195],"four":[104],"dimensional":[105],"rating":[106,144,175,185],"expressivity,":[108],"naturalness,":[109],"pronunciation":[110],"goodness":[111],"engagement.":[113],"goal":[115],"application":[118],"reliably":[121],"individual":[124],"annotator":[125,153],"which":[127],"used":[130],"address":[132],"issues":[133],"cognitive":[135],"load":[136],"as":[140,142],"well":[141],"cost.":[145],"initially":[147],"train":[148],"baseline":[150],"directly":[151],"predicting":[152],"compare":[159],"it":[160],"under":[164],"three":[165],"different":[166],"settings":[167],"assuming:":[168],"(i)":[169],"each":[170],"entry":[171],"independent":[177],"others,":[179],"(ii)":[180],"joint":[182],"distribution":[183],"among":[184],"dimensions":[186],"exists,":[187],"(iii)":[188],"partial":[190],"set":[191],"remaining":[197],"entries":[198],"available.":[200]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2020,"cited_by_count":5},{"year":2017,"cited_by_count":2}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}