{"id":"https://openalex.org/W3177059608","doi":"https://doi.org/10.1109/qomex51781.2021.9465382","title":"On Inter-Rater Reliability for Crowdsourced QoE","display_name":"On Inter-Rater Reliability for Crowdsourced QoE","publication_year":2021,"publication_date":"2021-06-14","ids":{"openalex":"https://openalex.org/W3177059608","doi":"https://doi.org/10.1109/qomex51781.2021.9465382","mag":"3177059608"},"language":"en","primary_location":{"id":"doi:10.1109/qomex51781.2021.9465382","is_oa":false,"landing_page_url":"https://doi.org/10.1109/qomex51781.2021.9465382","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 13th International Conference on Quality of Multimedia Experience (QoMEX)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://opus.bibliothek.uni-augsburg.de/opus4/files/107329/107329.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5025794971","display_name":"Tobias Ho\u00dffeld","orcid":"https://orcid.org/0000-0003-0173-595X"},"institutions":[{"id":"https://openalex.org/I25974101","display_name":"University of W\u00fcrzburg","ror":"https://ror.org/00fbnyb24","country_code":"DE","type":"education","lineage":["https://openalex.org/I25974101"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Tobias Ho\u00dffeld","raw_affiliation_strings":["University of Würzburg,Chair of Communication Networks,Würzburg,Germany"],"affiliations":[{"raw_affiliation_string":"University of Würzburg,Chair of Communication Networks,Würzburg,Germany","institution_ids":["https://openalex.org/I25974101"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010332473","display_name":"Michael Seufert","orcid":"https://orcid.org/0000-0002-5036-5206"},"institutions":[{"id":"https://openalex.org/I25974101","display_name":"University of W\u00fcrzburg","ror":"https://ror.org/00fbnyb24","country_code":"DE","type":"education","lineage":["https://openalex.org/I25974101"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Michael Seufert","raw_affiliation_strings":["University of Würzburg,Chair of Communication Networks,Würzburg,Germany"],"affiliations":[{"raw_affiliation_string":"University of Würzburg,Chair of Communication Networks,Würzburg,Germany","institution_ids":["https://openalex.org/I25974101"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069905492","display_name":"Babak Naderi","orcid":"https://orcid.org/0009-0006-4778-5417"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Babak Naderi","raw_affiliation_strings":["Technische Universität Berlin,Quality and Usability Lab,Berlin,Germany"],"affiliations":[{"raw_affiliation_string":"Technische Universität Berlin,Quality and Usability Lab,Berlin,Germany","institution_ids":["https://openalex.org/I4577782"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5025794971"],"corresponding_institution_ids":["https://openalex.org/I25974101"],"apc_list":null,"apc_paid":null,"fwci":0.728,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.76599749,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"3","issue":null,"first_page":"37","last_page":"42"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9836000204086304,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9750000238418579,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9208613634109497},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.7385252714157104},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7152096033096313},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6537048816680908},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5500137805938721},{"id":"https://openalex.org/keywords/moderation","display_name":"Moderation","score":0.5011217594146729},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.4774656593799591},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.45292770862579346},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.44173464179039},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.42992717027664185},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.42565980553627014},{"id":"https://openalex.org/keywords/sensitivity","display_name":"Sensitivity (control systems)","score":0.42507049441337585},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.40845996141433716},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3761669397354126},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.344684898853302},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14170381426811218},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1059466302394867},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.10426521301269531}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9208613634109497},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7385252714157104},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7152096033096313},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6537048816680908},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5500137805938721},{"id":"https://openalex.org/C93225998","wikidata":"https://www.wikidata.org/wiki/Q1941972","display_name":"Moderation","level":2,"score":0.5011217594146729},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.4774656593799591},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.45292770862579346},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.44173464179039},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.42992717027664185},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.42565980553627014},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.42507049441337585},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40845996141433716},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3761669397354126},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.344684898853302},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14170381426811218},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1059466302394867},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.10426521301269531},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C24326235","wikidata":"https://www.wikidata.org/wiki/Q126095","display_name":"Electronic engineering","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/qomex51781.2021.9465382","is_oa":false,"landing_page_url":"https://doi.org/10.1109/qomex51781.2021.9465382","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 13th International Conference on Quality of Multimedia Experience (QoMEX)","raw_type":"proceedings-article"},{"id":"pmh:oai:uni-augsburg.opus-bayern.de:107329","is_oa":true,"landing_page_url":"https://nbn-resolving.org/urn:nbn:de:bvb:384-opus4-1073296","pdf_url":"https://opus.bibliothek.uni-augsburg.de/opus4/files/107329/107329.pdf","source":{"id":"https://openalex.org/S4306400930","display_name":"OPUS (Augsburg University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I119916105","host_organization_name":"Augsburg University","host_organization_lineage":["https://openalex.org/I119916105"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"doc-type:conferenceobject"}],"best_oa_location":{"id":"pmh:oai:uni-augsburg.opus-bayern.de:107329","is_oa":true,"landing_page_url":"https://nbn-resolving.org/urn:nbn:de:bvb:384-opus4-1073296","pdf_url":"https://opus.bibliothek.uni-augsburg.de/opus4/files/107329/107329.pdf","source":{"id":"https://openalex.org/S4306400930","display_name":"OPUS (Augsburg University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I119916105","host_organization_name":"Augsburg University","host_organization_lineage":["https://openalex.org/I119916105"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"doc-type:conferenceobject"},"sustainable_development_goals":[{"score":0.6000000238418579,"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3177059608.pdf","grobid_xml":"https://content.openalex.org/works/W3177059608.grobid-xml"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W140775045","https://openalex.org/W745544473","https://openalex.org/W1595637775","https://openalex.org/W1970543122","https://openalex.org/W1975879668","https://openalex.org/W2013861934","https://openalex.org/W2025518782","https://openalex.org/W2053154970","https://openalex.org/W2061504941","https://openalex.org/W2124379907","https://openalex.org/W2141403362","https://openalex.org/W2154999911","https://openalex.org/W2155243985","https://openalex.org/W2160842878","https://openalex.org/W2163031058","https://openalex.org/W2164777277","https://openalex.org/W2177315755","https://openalex.org/W2325963232","https://openalex.org/W2395622121","https://openalex.org/W2566053671","https://openalex.org/W2963562284","https://openalex.org/W3015717277","https://openalex.org/W3037833165","https://openalex.org/W3090476742","https://openalex.org/W3100211355","https://openalex.org/W3107070581","https://openalex.org/W4237478516","https://openalex.org/W6653998682","https://openalex.org/W6712010412"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W2113265763"],"abstract_inverted_index":{"Crowdsourcing":[0],"offers":[1],"a":[2,35,143],"faster,":[3],"cheaper,":[4],"and":[5,30,93,114,126],"more":[6],"scalable":[7],"approach":[8],"than":[9],"the":[10,19,45,60,78,84,101,112,120,123,132,149,174],"traditional":[11,108],"laboratory":[12],"quality":[13,75,180],"assessment":[14],"tests.":[15,76],"However,":[16],"participants":[17],"perform":[18],"test":[20,36],"in":[21,63,118,169],"their":[22,27,66,116,166],"own":[23,28],"working":[24],"environment,":[25],"using":[26],"hardware":[29],"without":[31],"direct":[32],"supervision":[33],"of":[34,42,65,72,86,96,103,122,176],"moderator,":[37],"leading":[38],"to":[39,59,68,164,171],"different":[40,87,91],"types":[41],"biases":[43,125],"on":[44,111],"ratings.":[46],"In":[47],"this":[48,159],"paper,":[49],"we":[50,82],"compare":[51,115],"several":[52],"reliability":[53,109,175],"metrics":[54,110],"that":[55,131,154],"are":[56],"commonly":[57],"applied":[58],"subjective":[61],"ratings":[62,85,113],"terms":[64],"sensitivity":[67,117],"identify":[69],"typical":[70],"issues":[71,147],"crowdsourced":[73],"media":[74],"Following":[77],"subject":[79],"bias":[80,92],"theory,":[81],"simulate":[83],"user":[88],"groups":[89],"with":[90,148],"various":[94],"magnitudes":[95],"uncertainty,":[97],"while":[98],"also":[99],"considering":[100],"presence":[102],"unreliable":[104],"raters.":[105],"We":[106],"apply":[107],"identifying":[119],"severity":[121],"raters'":[124],"uncertainties.":[127],"Our":[128],"results":[129,177],"show":[130],"average":[133],"Spearman's":[134],"rank":[135],"correlation":[136],"coefficient":[137],"between":[138],"raters":[139],"can":[140],"serve":[141],"as":[142],"strong":[144],"indicator":[145],"for":[146,158],"crowdsourcing":[150],"study.":[151],"This":[152],"means":[153],"scoring":[155],"too":[156],"low":[157],"metric":[160],"should":[161],"encourage":[162],"researchers":[163],"revisit":[165],"study":[167],"design":[168],"order":[170],"eventually":[172],"improve":[173],"from":[178],"crowdsourcing-based":[179],"studies.":[181]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-03-10T16:38:18.471706","created_date":"2025-10-10T00:00:00"}