{"id":"https://openalex.org/W4301184860","doi":"https://doi.org/10.1109/qomex55416.2022.9900894","title":"Evaluating the Robustness of Speech Evaluation Standards for the Crowd","display_name":"Evaluating the Robustness of Speech Evaluation Standards for the Crowd","publication_year":2022,"publication_date":"2022-09-05","ids":{"openalex":"https://openalex.org/W4301184860","doi":"https://doi.org/10.1109/qomex55416.2022.9900894"},"language":"en","primary_location":{"id":"doi:10.1109/qomex55416.2022.9900894","is_oa":false,"landing_page_url":"https://doi.org/10.1109/qomex55416.2022.9900894","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 14th International Conference on Quality of Multimedia Experience (QoMEX)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058473354","display_name":"Edwin Gamboa","orcid":"https://orcid.org/0000-0002-5037-5279"},"institutions":[{"id":"https://openalex.org/I119449181","display_name":"Technische Universit\u00e4t Ilmenau","ror":"https://ror.org/01weqhp73","country_code":"DE","type":"education","lineage":["https://openalex.org/I119449181"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Edwin Gamboa","raw_affiliation_strings":["User-centric Analysis of Multimedia Data Group, TU Ilmenau,Ilmenau,Germany","User-centric Analysis of Multimedia Data Group, TU Ilmenau, Ilmenau, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"User-centric Analysis of Multimedia Data Group, TU Ilmenau,Ilmenau,Germany","institution_ids":["https://openalex.org/I119449181"]},{"raw_affiliation_string":"User-centric Analysis of Multimedia Data Group, TU Ilmenau, Ilmenau, Germany","institution_ids":["https://openalex.org/I119449181"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069905492","display_name":"Babak Naderi","orcid":"https://orcid.org/0009-0006-4778-5417"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Babak Naderi","raw_affiliation_strings":["Quality and Usability Lab,TU Berlin,Berlin,Germany","TU Berlin, Quality and Usability Lab, Berlin, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Quality and Usability Lab,TU Berlin,Berlin,Germany","institution_ids":["https://openalex.org/I4577782"]},{"raw_affiliation_string":"TU Berlin, Quality and Usability Lab, Berlin, Germany","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041286648","display_name":"Matthias Hirth","orcid":"https://orcid.org/0000-0002-1359-363X"},"institutions":[{"id":"https://openalex.org/I119449181","display_name":"Technische Universit\u00e4t Ilmenau","ror":"https://ror.org/01weqhp73","country_code":"DE","type":"education","lineage":["https://openalex.org/I119449181"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Matthias Hirth","raw_affiliation_strings":["User-centric Analysis of Multimedia Data Group, TU Ilmenau,Ilmenau,Germany","User-centric Analysis of Multimedia Data Group, TU Ilmenau, Ilmenau, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"User-centric Analysis of Multimedia Data Group, TU Ilmenau,Ilmenau,Germany","institution_ids":["https://openalex.org/I119449181"]},{"raw_affiliation_string":"User-centric Analysis of Multimedia Data Group, TU Ilmenau, Ilmenau, Germany","institution_ids":["https://openalex.org/I119449181"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075001284","display_name":"Sebastian M\u00f6ller","orcid":"https://orcid.org/0000-0003-3057-0760"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sebastian Moller","raw_affiliation_strings":["Quality and Usability Lab,TU Berlin,Berlin,Germany","TU Berlin, Quality and Usability Lab, Berlin, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Quality and Usability Lab,TU Berlin,Berlin,Germany","institution_ids":["https://openalex.org/I4577782"]},{"raw_affiliation_string":"TU Berlin, Quality and Usability Lab, Berlin, Germany","institution_ids":["https://openalex.org/I4577782"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"3","issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9804999828338623,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9750000238418579,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.8254974484443665},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7165203094482422},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4696311950683594}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.8254974484443665},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7165203094482422},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4696311950683594},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/qomex55416.2022.9900894","is_oa":false,"landing_page_url":"https://doi.org/10.1109/qomex55416.2022.9900894","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 14th International Conference on Quality of Multimedia Experience (QoMEX)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Good health and well-being","id":"https://metadata.un.org/sdg/3","score":0.5199999809265137}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W2132481689","https://openalex.org/W2163031058","https://openalex.org/W2341853547","https://openalex.org/W2395622121","https://openalex.org/W2395718496","https://openalex.org/W2408491799","https://openalex.org/W2890184085","https://openalex.org/W3015717277","https://openalex.org/W3025844872","https://openalex.org/W3092394385","https://openalex.org/W3100211355","https://openalex.org/W3107070581","https://openalex.org/W3177059608","https://openalex.org/W4225905067","https://openalex.org/W6704415283","https://openalex.org/W6711846516","https://openalex.org/W6777181074","https://openalex.org/W6782481672","https://openalex.org/W6784135118"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W4402327032","https://openalex.org/W2382290278"],"abstract_inverted_index":{"Subjective":[0],"assessments":[1,12],"are":[2,13,200,219,232],"a":[3,52,60,175,185,189,203,225],"key":[4],"component":[5],"of":[6,65,86,110,177],"speech":[7,34,123,132,195,273],"quality":[8,35,124,133,165,196],"research.":[9],"Traditionally,":[10],"the":[11,30,70,82,87,93,108,111,138,163,178,182,229,240,252,261],"conducted":[14],"in":[15,17,72,98,129,141,192,202,272],"laboratories":[16],"controlled":[18],"conditions":[19],"and":[20,62,67,84,126,181,194,254],"following":[21],"international":[22],"standards":[23],"like":[24],"ITU-T":[25,112,139,262],"Rec.P.800.":[26],"However,":[27,135],"even":[28,50,136],"before":[29],"current":[31],"pandemic,":[32],"more":[33],"research":[36],"used":[37,201],"crowdsourcing-based":[38],"approaches":[39],"for":[40],"collecting":[41],"subjective":[42,88],"ratings.":[43],"Crowdsourcing":[44],"allows":[45],"researchers":[46],"to":[47,56,68,103,152,161,221,234,239,267],"collect":[48,57],"data":[49,58,96,164,223],"without":[51],"dedicated":[53],"test":[54],"laboratory,":[55],"from":[59,122,174,212,224],"huge":[61],"diverse":[63],"group":[64],"participants,":[66],"perform":[69],"assessment":[71],"various":[73],"real-life":[74],"settings.":[75],"Still,":[76],"this":[77,155,158],"approach":[78,104],"raises":[79],"questions":[80],"about":[81,208],"reliability":[83],"validity":[85],"ratings,":[89],"especially":[90],"when":[91],"comparing":[92],"ratings":[94,218],"with":[95,137,187,207,251,255],"collected":[97,217],"standardized":[99],"procedures.":[100],"One":[101,171],"step":[102],"these":[105],"challenges":[106],"was":[107],"development":[109],"Rec.P.808":[113,140,263],"standard.":[114,156],"This":[115],"standard":[116],"helps":[117],"practitioners":[118],"implement":[119],"best":[120],"practices":[121],"studies":[125,128],"crowdsourcing":[127,193,205],"their":[130],"crowdsourced":[131],"assessments.":[134,197],"action,":[142],"it":[143],"is":[144,150,173,184,264],"unclear":[145],"how":[146],"much":[147],"background":[148,191],"knowledge":[149],"necessary":[151],"successfully":[153],"\u201cimplement\u201d":[154],"Therefore,":[157],"paper":[159],"aims":[160],"assess":[162],"differences":[166],"between":[167],"two":[168,209,230],"P.808":[169,179],"implementations.":[170],"implementation":[172],"co-author":[176],"standard,":[180],"other":[183],"researcher":[186],"only":[188],"little":[190],"Both":[198],"implementations":[199,231,248],"large-scale":[204],"study":[206],"hundred":[210],"users":[211],"Amazon":[213],"Mechanical":[214],"Turk.":[215],"The":[216,243],"compared":[220,233],"gold-standard":[222],"certified":[226],"laboratory.":[227],"Also,":[228],"analyze":[235],"whether":[236],"they":[237],"lead":[238],"same":[241],"conclusions.":[242],"results":[244],"show":[245],"that":[246,260],"both":[247],"correlate":[249],"strongly":[250],"laboratory":[253],"each":[256],"other.":[257],"Thus,":[258],"suggesting":[259],"robust":[265],"enough":[266],"be":[268],"implemented":[269],"by":[270],"non-experts":[271],"evaluation":[274],"or":[275],"crowdsourcing.":[276]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}