{"id":"https://openalex.org/W2945648216","doi":"https://doi.org/10.1145/3308560.3317084","title":"Intra- and Inter-rater Agreement in a Subjective Speech Quality Assessment Task in Crowdsourcing","display_name":"Intra- and Inter-rater Agreement in a Subjective Speech Quality Assessment Task in Crowdsourcing","publication_year":2019,"publication_date":"2019-05-13","ids":{"openalex":"https://openalex.org/W2945648216","doi":"https://doi.org/10.1145/3308560.3317084","mag":"2945648216"},"language":"en","primary_location":{"id":"doi:10.1145/3308560.3317084","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308560.3317084","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of The 2019 World Wide Web Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3308560.3317084","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103913459","display_name":"Rafael Zequeira Jim\u00e9nez","orcid":null},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Rafael Zequeira Jim\u00e9nez","raw_affiliation_strings":["TU Berlin"],"affiliations":[{"raw_affiliation_string":"TU Berlin","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088133485","display_name":"Anna Llagostera","orcid":null},"institutions":[{"id":"https://openalex.org/I205156485","display_name":"Rohde & Schwarz (Germany)","ror":"https://ror.org/048j11q41","country_code":"DE","type":"company","lineage":["https://openalex.org/I205156485"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Anna Llagostera","raw_affiliation_strings":["Rohde & Schwarz SwissQual AG,"],"affiliations":[{"raw_affiliation_string":"Rohde & Schwarz SwissQual AG,","institution_ids":["https://openalex.org/I205156485"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069905492","display_name":"Babak Naderi","orcid":"https://orcid.org/0009-0006-4778-5417"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Babak Naderi","raw_affiliation_strings":["TU Berlin,"],"affiliations":[{"raw_affiliation_string":"TU Berlin,","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075001284","display_name":"Sebastian M\u00f6ller","orcid":"https://orcid.org/0000-0003-3057-0760"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sebastian M\u00f6ller","raw_affiliation_strings":["TU Berlin,"],"affiliations":[{"raw_affiliation_string":"TU Berlin,","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013879623","display_name":"Jens Berger","orcid":"https://orcid.org/0000-0002-8978-5099"},"institutions":[{"id":"https://openalex.org/I205156485","display_name":"Rohde & Schwarz (Germany)","ror":"https://ror.org/048j11q41","country_code":"DE","type":"company","lineage":["https://openalex.org/I205156485"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Jens Berger","raw_affiliation_strings":["Rohde & Schwarz SwissQual AG,"],"affiliations":[{"raw_affiliation_string":"Rohde & Schwarz SwissQual AG,","institution_ids":["https://openalex.org/I205156485"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5103913459"],"corresponding_institution_ids":["https://openalex.org/I4577782"],"apc_list":null,"apc_paid":null,"fwci":2.1867,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.88787174,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1138","last_page":"1143"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9401999711990356,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.90420001745224,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.945701003074646},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"Inter-rater reliability","score":0.6284036040306091},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6063673496246338},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5970910787582397},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.5966461300849915},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5613188147544861},{"id":"https://openalex.org/keywords/quality-assessment","display_name":"Quality assessment","score":0.4161291718482971},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.36389368772506714},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.35881543159484863},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.300506591796875},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.1457889974117279},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.1256571114063263},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.09173056483268738},{"id":"https://openalex.org/keywords/rating-scale","display_name":"Rating scale","score":0.07698583602905273},{"id":"https://openalex.org/keywords/evaluation-methods","display_name":"Evaluation methods","score":0.07158008217811584},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07043766975402832},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.0602191686630249},{"id":"https://openalex.org/keywords/developmental-psychology","display_name":"Developmental psychology","score":0.059280216693878174}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.945701003074646},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.6284036040306091},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6063673496246338},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5970910787582397},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.5966461300849915},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5613188147544861},{"id":"https://openalex.org/C3020001037","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assessment","level":3,"score":0.4161291718482971},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.36389368772506714},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.35881543159484863},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.300506591796875},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.1457889974117279},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.1256571114063263},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.09173056483268738},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.07698583602905273},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.07158008217811584},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07043766975402832},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0602191686630249},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.059280216693878174},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3308560.3317084","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308560.3317084","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of The 2019 World Wide Web Conference","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3308560.3317084","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308560.3317084","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of The 2019 World Wide Web Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W199640592","https://openalex.org/W1228173857","https://openalex.org/W1595637775","https://openalex.org/W1972182967","https://openalex.org/W2013861934","https://openalex.org/W2082540701","https://openalex.org/W2091996826","https://openalex.org/W2104626092","https://openalex.org/W2141403362","https://openalex.org/W2160473997","https://openalex.org/W2330836443","https://openalex.org/W2395622121","https://openalex.org/W2408491799","https://openalex.org/W2802553787","https://openalex.org/W2890184085","https://openalex.org/W2964092706"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3178669050"],"abstract_inverted_index":{"Crowdsourcing":[0],"is":[1,24,29],"a":[2,49,58,88,190],"great":[3],"tool":[4],"for":[5],"conducting":[6],"subjective":[7,50],"user":[8],"studies":[9],"with":[10,78,93],"large":[11],"amounts":[12],"of":[13,21,30,180,192],"users.":[14],"Collecting":[15],"reliable":[16],"annotations":[17],"about":[18],"the":[19,43,64,94,107,120,123,129,133,144,155,159,161,164,167,178,181],"quality":[20,52],"speech":[22,51,76,102],"stimuli":[23,77],"challenging.":[25],"The":[26,101],"task":[27,182],"itself":[28],"high":[31],"subjectivity":[32,179],"and":[33,45,66,98,114,136,183],"users":[34],"in":[35,63,67,69,91,132,147],"crowdsourcing":[36,68,124],"work":[37,41],"without":[38],"supervision.":[39],"This":[40],"investigates":[42],"intra-":[44],"inter-listener":[46],"agreement":[47,157],"withing":[48],"assessment":[53,165],"task.":[54],"To":[55],"this":[56],"end,":[57],"study":[59,173],"has":[60],"been":[61],"conducted":[62,163],"laboratory":[65],"which":[70],"listeners":[71,162],"were":[72,85,104,115,126],"requested":[73],"to":[74,80,119,128,176,194],"rate":[75],"respect":[79],"their":[81],"overall":[82],"quality.":[83],"Ratings":[84],"collected":[86,131,146],"on":[87],"5-point":[89],"scale":[90],"accordance":[92],"ITU-T":[95,109],"Rec.":[96,110],"P.800":[97],"P.808,":[99],"respectively.":[100],"samples":[103],"taken":[105],"from":[106],"database":[108],"P.501":[111],"Annex":[112],"D,":[113],"presented":[116],"four":[117],"times":[118],"listeners.":[121],"Finally,":[122],"results":[125],"contrasted":[127],"ratings":[130,145],"laboratory.":[134],"Strong":[135],"significant":[137],"Spearman\u2019s":[138],"correlation":[139],"was":[140],"achieved":[141],"when":[142],"contrasting":[143],"both":[148],"environments.":[149],"Our":[150,172],"analysis":[151],"show":[152],"that":[153,186],"while":[154],"inter-rater":[156],"increased":[158],"more":[160],"task,":[166],"intra-rater":[168],"reliability":[169],"remained":[170],"constant.":[171],"setup":[174],"helped":[175],"overcome":[177],"we":[184],"found":[185],"disagreement":[187],"can":[188],"represent":[189],"source":[191],"information":[193],"some":[195],"extent.":[196]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1}],"updated_date":"2026-04-17T18:11:37.981687","created_date":"2025-10-10T00:00:00"}