{"id":"https://openalex.org/W2163031058","doi":"https://doi.org/10.1109/tmm.2013.2291663","title":"Best Practices for QoE Crowdtesting: QoE Assessment With Crowdsourcing","display_name":"Best Practices for QoE Crowdtesting: QoE Assessment With Crowdsourcing","publication_year":2014,"publication_date":"2014-01-24","ids":{"openalex":"https://openalex.org/W2163031058","doi":"https://doi.org/10.1109/tmm.2013.2291663","mag":"2163031058"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2013.2291663","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2013.2291663","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5025794971","display_name":"Tobias Ho\u00dffeld","orcid":"https://orcid.org/0000-0003-0173-595X"},"institutions":[{"id":"https://openalex.org/I25974101","display_name":"University of W\u00fcrzburg","ror":"https://ror.org/00fbnyb24","country_code":"DE","type":"education","lineage":["https://openalex.org/I25974101"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Tobias Hossfeld","raw_affiliation_strings":["University of W\u00fcrzburg, Institute of Computer Science, W\u00fcrzburg, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of W\u00fcrzburg, Institute of Computer Science, W\u00fcrzburg, Germany","institution_ids":["https://openalex.org/I25974101"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074400591","display_name":"Christian Keimel","orcid":"https://orcid.org/0000-0002-4234-7745"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Christian Keimel","raw_affiliation_strings":["Technische Universit\u00e4t M\u00fcnchen (TUM), Institute for Data Processing, M\u00fcnchen, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technische Universit\u00e4t M\u00fcnchen (TUM), Institute for Data Processing, M\u00fcnchen, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041286648","display_name":"Matthias Hirth","orcid":"https://orcid.org/0000-0002-1359-363X"},"institutions":[{"id":"https://openalex.org/I25974101","display_name":"University of W\u00fcrzburg","ror":"https://ror.org/00fbnyb24","country_code":"DE","type":"education","lineage":["https://openalex.org/I25974101"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Matthias Hirth","raw_affiliation_strings":["University of W\u00fcrzburg, Institute of Computer Science, W\u00fcrzburg, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of W\u00fcrzburg, Institute of Computer Science, W\u00fcrzburg, Germany","institution_ids":["https://openalex.org/I25974101"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090942436","display_name":"Bruno Gardlo","orcid":null},"institutions":[{"id":"https://openalex.org/I4210122474","display_name":"Forschungszentrum Telekommunikation Wien","ror":"https://ror.org/026nftk33","country_code":"AT","type":"facility","lineage":["https://openalex.org/I4210122474"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Bruno Gardlo","raw_affiliation_strings":["Telecommunications Research Center Vienna (FTW), Vienna, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Telecommunications Research Center Vienna (FTW), Vienna, Austria","institution_ids":["https://openalex.org/I4210122474"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021403095","display_name":"Julian Habigt","orcid":null},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Julian Habigt","raw_affiliation_strings":["Technische Universit\u00e4t M\u00fcnchen (TUM), Institute for Data Processing, M\u00fcnchen, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technische Universit\u00e4t M\u00fcnchen (TUM), Institute for Data Processing, M\u00fcnchen, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074346065","display_name":"Klaus Diepold","orcid":"https://orcid.org/0000-0003-0439-7511"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Klaus Diepold","raw_affiliation_strings":["Technische Universit\u00e4t M\u00fcnchen (TUM), Institute for Data Processing, M\u00fcnchen, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technische Universit\u00e4t M\u00fcnchen (TUM), Institute for Data Processing, M\u00fcnchen, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111979857","display_name":"Phuoc Tran\u2010Gia","orcid":null},"institutions":[{"id":"https://openalex.org/I25974101","display_name":"University of W\u00fcrzburg","ror":"https://ror.org/00fbnyb24","country_code":"DE","type":"education","lineage":["https://openalex.org/I25974101"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Phuoc Tran-Gia","raw_affiliation_strings":["University of W\u00fcrzburg, Institute of Computer Science, W\u00fcrzburg, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of W\u00fcrzburg, Institute of Computer Science, W\u00fcrzburg, Germany","institution_ids":["https://openalex.org/I25974101"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":55.442,"has_fulltext":false,"cited_by_count":281,"citation_normalized_percentile":{"value":0.99907304,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"16","issue":"2","first_page":"541","last_page":"558"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.9639999866485596,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8355911374092102},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8254852294921875},{"id":"https://openalex.org/keywords/quality-of-experience","display_name":"Quality of experience","score":0.7952917814254761},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6069708466529846},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5570580959320068},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.48683232069015503},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.45266374945640564},{"id":"https://openalex.org/keywords/incentive","display_name":"Incentive","score":0.41377508640289307},{"id":"https://openalex.org/keywords/subjective-video-quality","display_name":"Subjective video quality","score":0.4131254255771637},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.17973843216896057},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.14215174317359924},{"id":"https://openalex.org/keywords/quality-of-service","display_name":"Quality of service","score":0.14032945036888123},{"id":"https://openalex.org/keywords/image-quality","display_name":"Image quality","score":0.12527072429656982},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.12506523728370667}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8355911374092102},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8254852294921875},{"id":"https://openalex.org/C2779333187","wikidata":"https://www.wikidata.org/wiki/Q3132648","display_name":"Quality of experience","level":3,"score":0.7952917814254761},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6069708466529846},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5570580959320068},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.48683232069015503},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.45266374945640564},{"id":"https://openalex.org/C29122968","wikidata":"https://www.wikidata.org/wiki/Q1414816","display_name":"Incentive","level":2,"score":0.41377508640289307},{"id":"https://openalex.org/C114227958","wikidata":"https://www.wikidata.org/wiki/Q7631422","display_name":"Subjective video quality","level":4,"score":0.4131254255771637},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.17973843216896057},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.14215174317359924},{"id":"https://openalex.org/C5119721","wikidata":"https://www.wikidata.org/wiki/Q220501","display_name":"Quality of service","level":2,"score":0.14032945036888123},{"id":"https://openalex.org/C55020928","wikidata":"https://www.wikidata.org/wiki/Q3813865","display_name":"Image quality","level":3,"score":0.12527072429656982},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.12506523728370667},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2013.2291663","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2013.2291663","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":59,"referenced_works":["https://openalex.org/W156959604","https://openalex.org/W279631868","https://openalex.org/W1525599330","https://openalex.org/W1599116104","https://openalex.org/W1737537661","https://openalex.org/W1779843715","https://openalex.org/W1857127940","https://openalex.org/W1966409995","https://openalex.org/W1988584482","https://openalex.org/W2013861934","https://openalex.org/W2024166834","https://openalex.org/W2025518782","https://openalex.org/W2044885272","https://openalex.org/W2049208969","https://openalex.org/W2061504941","https://openalex.org/W2065100127","https://openalex.org/W2075023066","https://openalex.org/W2096848877","https://openalex.org/W2097090466","https://openalex.org/W2097246133","https://openalex.org/W2098865355","https://openalex.org/W2101086718","https://openalex.org/W2104749423","https://openalex.org/W2105187896","https://openalex.org/W2106489024","https://openalex.org/W2107756083","https://openalex.org/W2122261143","https://openalex.org/W2124259158","https://openalex.org/W2132232498","https://openalex.org/W2134599218","https://openalex.org/W2135626977","https://openalex.org/W2135799691","https://openalex.org/W2138369874","https://openalex.org/W2141282920","https://openalex.org/W2143280940","https://openalex.org/W2147254635","https://openalex.org/W2151401338","https://openalex.org/W2157454221","https://openalex.org/W2158880898","https://openalex.org/W2159393007","https://openalex.org/W2160473997","https://openalex.org/W2165277686","https://openalex.org/W2170931744","https://openalex.org/W2171913066","https://openalex.org/W2187291759","https://openalex.org/W2187912528","https://openalex.org/W3121522380","https://openalex.org/W6606258704","https://openalex.org/W6610117260","https://openalex.org/W6631297879","https://openalex.org/W6635786100","https://openalex.org/W6637758838","https://openalex.org/W6639127845","https://openalex.org/W6674368962","https://openalex.org/W6674748245","https://openalex.org/W6683123733","https://openalex.org/W6686812563","https://openalex.org/W6686981222","https://openalex.org/W6990166387"],"related_works":["https://openalex.org/W4310908637","https://openalex.org/W1553085267","https://openalex.org/W3037266308","https://openalex.org/W2011253882","https://openalex.org/W1809225384","https://openalex.org/W2960369171","https://openalex.org/W2620623858","https://openalex.org/W2061256681","https://openalex.org/W2345171086","https://openalex.org/W2009291387"],"abstract_inverted_index":{"Quality":[0],"of":[1,29,60,83,113,118,127,181,196,203,212],"Experience":[2],"(QoE)":[3],"in":[4,23,54,66,77,88,147,154,192,207],"multimedia":[5],"applications":[6],"is":[7,206],"closely":[8],"linked":[9],"to":[10,25,41,99,135,143,164,170,237],"the":[11,27,36,67,73,100,111,116,123,128,131,148,155,210,224],"end":[12],"users'":[13],"perception":[14],"and":[15,70,80,94,122,140,151,169,214],"therefore":[16,177],"its":[17],"assessment":[18,43,219],"requires":[19],"subjective":[20,51],"user":[21,90,114,167],"studies":[22],"order":[24,134],"evaluate":[26],"degree":[28],"delight":[30],"or":[31],"annoyance":[32],"as":[33,220],"experienced":[34],"by":[35],"users.":[37],"QoE":[38,42,61,108,198,233],"crowdtesting":[39,62,109,199,234],"refers":[40],"using":[44],"crowdsourcing,":[45],"where":[46],"anonymous":[47],"test":[48,102,149,157],"subjects":[49],"conduct":[50],"tests":[52,129],"remotely":[53],"their":[55],"preferred":[56],"environment.":[57],"The":[58,201],"advantages":[59],"lie":[63],"not":[64],"only":[65],"reduced":[68],"time":[69],"costs":[71],"for":[72,223],"tests,":[74],"but":[75],"also":[76,152],"a":[78,179,193],"large":[79,194],"diverse":[81],"panel":[82],"international,":[84],"geographically":[85],"distributed":[86],"users":[87],"realistic":[89],"settings.":[91,103],"However,":[92],"conceptual":[93],"technical":[95],"challenges":[96],"emerge":[97],"due":[98],"remote":[101],"Key":[104],"issues":[105,186],"arising":[106],"from":[107],"include":[110],"reliability":[112,213],"ratings,":[115],"influence":[117],"incentives,":[119],"payment":[120],"schemes":[121],"unknown":[124],"environmental":[125],"context":[126],"on":[130,188,209],"results.":[132,240],"In":[133],"counter":[136],"these":[137,185],"issues,":[138],"strategies":[139],"methods":[141,161],"need":[142],"be":[144],"developed,":[145],"included":[146],"design,":[150],"implemented":[153],"actual":[156],"campaign,":[158],"while":[159],"statistical":[160],"are":[162],"required":[163],"identify":[165],"reliable":[166,239],"ratings":[168],"ensure":[171],"high":[172],"data":[173],"quality.":[174],"This":[175],"contribution":[176],"provides":[178],"collection":[180],"best":[182,226],"practices":[183],"addressing":[184],"based":[187],"our":[189,230],"experience":[190],"gained":[191],"set":[195],"conducted":[197],"studies.":[200],"focus":[202],"this":[204],"article":[205],"particular":[208],"issue":[211],"we":[215],"use":[216],"video":[217],"quality":[218],"an":[221],"example":[222],"proposed":[225],"practices,":[227],"showing":[228],"that":[229],"recommended":[231],"two-stage":[232],"design":[235],"leads":[236],"more":[238]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":16},{"year":2022,"cited_by_count":17},{"year":2021,"cited_by_count":29},{"year":2020,"cited_by_count":25},{"year":2019,"cited_by_count":24},{"year":2018,"cited_by_count":23},{"year":2017,"cited_by_count":25},{"year":2016,"cited_by_count":34},{"year":2015,"cited_by_count":33},{"year":2014,"cited_by_count":27}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}