{"id":"https://openalex.org/W3107070581","doi":"https://doi.org/10.1007/s41233-020-00042-1","title":"Towards speech quality assessment using a crowdsourcing approach: evaluation of standardized methods","display_name":"Towards speech quality assessment using a crowdsourcing approach: evaluation of standardized methods","publication_year":2020,"publication_date":"2020-11-22","ids":{"openalex":"https://openalex.org/W3107070581","doi":"https://doi.org/10.1007/s41233-020-00042-1","mag":"3107070581"},"language":"en","primary_location":{"id":"doi:10.1007/s41233-020-00042-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s41233-020-00042-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s41233-020-00042-1.pdf","source":{"id":"https://openalex.org/S4210197207","display_name":"Quality and User Experience","issn_l":"2366-0139","issn":["2366-0139","2366-0147"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Quality and User Experience","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s41233-020-00042-1.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069905492","display_name":"Babak Naderi","orcid":"https://orcid.org/0009-0006-4778-5417"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Babak Naderi","raw_affiliation_strings":["Quality and Usability Lab, Technische Universit\u00e4t Berlin, Berlin, Germany"],"affiliations":[{"raw_affiliation_string":"Quality and Usability Lab, Technische Universit\u00e4t Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103913459","display_name":"Rafael Zequeira Jim\u00e9nez","orcid":null},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Rafael Zequeira Jim\u00e9nez","raw_affiliation_strings":["Quality and Usability Lab, Technische Universit\u00e4t Berlin, Berlin, Germany"],"affiliations":[{"raw_affiliation_string":"Quality and Usability Lab, Technische Universit\u00e4t Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041286648","display_name":"Matthias Hirth","orcid":"https://orcid.org/0000-0002-1359-363X"},"institutions":[{"id":"https://openalex.org/I119449181","display_name":"Technische Universit\u00e4t Ilmenau","ror":"https://ror.org/01weqhp73","country_code":"DE","type":"education","lineage":["https://openalex.org/I119449181"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Matthias Hirth","raw_affiliation_strings":["User-centric Analysis of Multimedia Data Group, Technische Universit\u00e4t Ilmenau, Ilmenau, Germany"],"affiliations":[{"raw_affiliation_string":"User-centric Analysis of Multimedia Data Group, Technische Universit\u00e4t Ilmenau, Ilmenau, Germany","institution_ids":["https://openalex.org/I119449181"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075001284","display_name":"Sebastian M\u00f6ller","orcid":"https://orcid.org/0000-0003-3057-0760"},"institutions":[{"id":"https://openalex.org/I33256026","display_name":"German Research Centre for Artificial Intelligence","ror":"https://ror.org/01ayc5b57","country_code":"DE","type":"funder","lineage":["https://openalex.org/I33256026"]},{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sebastian M\u00f6ller","raw_affiliation_strings":["Quality and Usability Lab, Technische Universit\u00e4t Berlin, Berlin, Germany","Speech and Language Technology, German Research Center for Artificial Intelligence (DFKI), Berlin, Germany"],"affiliations":[{"raw_affiliation_string":"Quality and Usability Lab, Technische Universit\u00e4t Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I4577782"]},{"raw_affiliation_string":"Speech and Language Technology, German Research Center for Artificial Intelligence (DFKI), Berlin, Germany","institution_ids":["https://openalex.org/I33256026"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078710314","display_name":"Florian Metzger","orcid":"https://orcid.org/0000-0002-1729-0943"},"institutions":[{"id":"https://openalex.org/I25974101","display_name":"University of W\u00fcrzburg","ror":"https://ror.org/00fbnyb24","country_code":"DE","type":"education","lineage":["https://openalex.org/I25974101"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Florian Metzger","raw_affiliation_strings":["Chair of Communication Networks, University of W\u00fcrzburg, W\u00fcrzburg, Germany"],"affiliations":[{"raw_affiliation_string":"Chair of Communication Networks, University of W\u00fcrzburg, W\u00fcrzburg, Germany","institution_ids":["https://openalex.org/I25974101"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025794971","display_name":"Tobias Ho\u00dffeld","orcid":"https://orcid.org/0000-0003-0173-595X"},"institutions":[{"id":"https://openalex.org/I25974101","display_name":"University of W\u00fcrzburg","ror":"https://ror.org/00fbnyb24","country_code":"DE","type":"education","lineage":["https://openalex.org/I25974101"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Tobias Ho\u00dffeld","raw_affiliation_strings":["Chair of Communication Networks, University of W\u00fcrzburg, W\u00fcrzburg, Germany"],"affiliations":[{"raw_affiliation_string":"Chair of Communication Networks, University of W\u00fcrzburg, W\u00fcrzburg, Germany","institution_ids":["https://openalex.org/I25974101"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5069905492"],"corresponding_institution_ids":["https://openalex.org/I4577782"],"apc_list":{"value":2590,"currency":"EUR","value_usd":3390},"apc_paid":{"value":2590,"currency":"EUR","value_usd":3390},"fwci":2.5904,"has_fulltext":true,"cited_by_count":22,"citation_normalized_percentile":{"value":0.9070452,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"6","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10283","display_name":"Hearing Loss and Rehabilitation","score":0.9898999929428101,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12146","display_name":"Power Line Communications and Noise","score":0.9653000235557556,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9867777228355408},{"id":"https://openalex.org/keywords/comparability","display_name":"Comparability","score":0.8113083839416504},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6741541624069214},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6700245141983032},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6602100133895874},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.47773119807243347},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.4712677001953125},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.4500754177570343},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.446760356426239},{"id":"https://openalex.org/keywords/active-listening","display_name":"Active listening","score":0.41463494300842285},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.19715279340744019},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.18361347913742065},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07459637522697449}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9867777228355408},{"id":"https://openalex.org/C197947376","wikidata":"https://www.wikidata.org/wiki/Q5155608","display_name":"Comparability","level":2,"score":0.8113083839416504},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6741541624069214},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6700245141983032},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6602100133895874},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.47773119807243347},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.4712677001953125},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.4500754177570343},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.446760356426239},{"id":"https://openalex.org/C177291462","wikidata":"https://www.wikidata.org/wiki/Q423038","display_name":"Active listening","level":2,"score":0.41463494300842285},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.19715279340744019},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.18361347913742065},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07459637522697449},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/s41233-020-00042-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s41233-020-00042-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s41233-020-00042-1.pdf","source":{"id":"https://openalex.org/S4210197207","display_name":"Quality and User Experience","issn_l":"2366-0139","issn":["2366-0139","2366-0147"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Quality and User Experience","raw_type":"journal-article"},{"id":"pmh:oai:www.db-thueringen.de:dbt_mods_00052042","is_oa":true,"landing_page_url":"https://www.db-thueringen.de/receive/dbt_mods_00052042","pdf_url":null,"source":{"id":"https://openalex.org/S4306400120","display_name":"Common Library Network (Der Gemeinsame Bibliotheksverbund)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210149878","host_organization_name":"Verbundzentrale des GBV","host_organization_lineage":["https://openalex.org/I4210149878"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1007/s41233-020-00042-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s41233-020-00042-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s41233-020-00042-1.pdf","source":{"id":"https://openalex.org/S4210197207","display_name":"Quality and User Experience","issn_l":"2366-0139","issn":["2366-0139","2366-0147"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Quality and User Experience","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320324094","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3107070581.pdf","grobid_xml":"https://content.openalex.org/works/W3107070581.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W745544473","https://openalex.org/W1876635246","https://openalex.org/W1969235627","https://openalex.org/W2022623421","https://openalex.org/W2040064936","https://openalex.org/W2049208969","https://openalex.org/W2051951610","https://openalex.org/W2135626977","https://openalex.org/W2160473997","https://openalex.org/W2163031058","https://openalex.org/W2345254477","https://openalex.org/W2395622121","https://openalex.org/W2395718496","https://openalex.org/W2408491799","https://openalex.org/W2474880636","https://openalex.org/W2521718083","https://openalex.org/W2572094800","https://openalex.org/W2756489727","https://openalex.org/W2756644488","https://openalex.org/W2759861411","https://openalex.org/W2806526132","https://openalex.org/W2890184085","https://openalex.org/W2892235646","https://openalex.org/W2908708742","https://openalex.org/W3015717277","https://openalex.org/W3018201762","https://openalex.org/W3025844872","https://openalex.org/W3037636137","https://openalex.org/W3037833165","https://openalex.org/W3099045918","https://openalex.org/W3100211355","https://openalex.org/W3101528112","https://openalex.org/W3182211633","https://openalex.org/W4233009235","https://openalex.org/W4233065845"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W1503094549","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W2365594754","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4287902769"],"abstract_inverted_index":{"Abstract":[0],"Subjective":[1],"speech":[2,43,67,111,172],"quality":[3,44,52,68,108,173],"assessment":[4],"has":[5],"traditionally":[6],"been":[7,87],"carried":[8,88],"out":[9,89],"in":[10,70,144],"laboratory":[11,132],"environments":[12],"under":[13],"controlled":[14],"conditions.":[15],"With":[16],"the":[17,34,51,55,97,106,110,114,121,124,128,137],"advent":[18],"of":[19,50,57,72,74,83,109,123,130],"crowdsourcing":[20,93,125],"platforms":[21,94],"tasks,":[22],"which":[23],"need":[24],"human":[25],"intelligence,":[26],"can":[27],"be":[28],"resolved":[29],"by":[30],"crowd":[31],"workers":[32],"over":[33],"Internet.":[35],"Crowdsourcing":[36],"also":[37],"offers":[38],"a":[39],"new":[40],"paradigm":[41],"for":[42,170],"assessment,":[45],"promising":[46],"higher":[47],"ecological":[48],"validity":[49],"judgments":[53],"at":[54],"expense":[56],"potentially":[58],"lower":[59],"reliability.":[60],"This":[61],"paper":[62],"compares":[63],"laboratory-based":[64],"and":[65,76,95,162],"crowdsourcing-based":[66,171],"assessments":[69],"terms":[71],"comparability":[73],"results":[75,122,129],"efficiency.":[77],"For":[78],"this":[79],"purpose,":[80],"three":[81,91],"pairs":[82],"listening-only":[84],"tests":[85,133],"have":[86],"using":[90],"different":[92],"following":[96,113],"ITU-T":[98,138],"Recommendation":[99,139],"P.808.":[100],"In":[101],"each":[102],"test,":[103],"listeners":[104],"judge":[105],"overall":[107],"sample":[112],"Absolute":[115],"Category":[116],"Rating":[117],"procedure.":[118],"We":[119],"compare":[120],"approach":[126],"with":[127,157],"standard":[131],"performed":[134],"according":[135],"to":[136,150,159],"P.800.":[140],"Results":[141],"show":[142],"that":[143,166],"most":[145],"cases,":[146],"both":[147],"paradigms":[148],"lead":[149],"comparable":[151],"results.":[152],"Notable":[153],"differences":[154],"are":[155,164],"discussed":[156],"respect":[158],"their":[160],"sources,":[161],"conclusions":[163],"drawn":[165],"establish":[167],"practical":[168],"guidelines":[169],"assessment.":[174]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":10},{"year":2020,"cited_by_count":1}],"updated_date":"2026-03-15T09:29:46.208133","created_date":"2025-10-10T00:00:00"}