{"id":"https://openalex.org/W2908862202","doi":"https://doi.org/10.1108/ejm-07-2017-0491","title":"A panel for lemons? Positivity bias, reputation systems and data quality on MTurk","display_name":"A panel for lemons? Positivity bias, reputation systems and data quality on MTurk","alternate_titles":[],"publication_year":2019,"publication_date":"2019-01-15","ids":{"openalex":"https://openalex.org/W2908862202","doi":"https://doi.org/10.1108/ejm-07-2017-0491","mag":"2908862202"},"language":"en","primary_location":{"id":"doi:10.1108/ejm-07-2017-0491","doi":"https://doi.org/10.1108/ejm-07-2017-0491","is_oa":false,"landing_page_url":"https://doi.org/10.1108/ejm-07-2017-0491","pdf_url":null,"source":{"id":"https://openalex.org/S89389284","display_name":"European Journal of Marketing","issn_l":"0309-0566","issn":["0309-0566","1758-7123"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a-star","anvur-class-a","anvur-scientific","cwts-core","erih-plus","fnege-2","jufo-2","norway-1","poland-70","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319811","host_organization_name":"Emerald Publishing Limited","host_organization_lineage":["https://openalex.org/P4310319811"],"host_organization_lineage_names":["Emerald Publishing Limited"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"European Journal of Marketing","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W2908862202"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026486449","display_name":"Ted Matherly","orcid":"https://orcid.org/0000-0002-8628-1004","observed_orcids":["https://orcid.org/0000-0002-8628-1004"]},"institutions":[{"id":"https://openalex.org/I115475287","display_name":"Oklahoma State University","ror":"https://ror.org/01g9vbr38","country_code":"US","type":"education","lineage":["https://openalex.org/I115475287"]}],"countries":["US"],"is_corresponding":true,"contributor_roles":[],"raw_author_name":"Ted Matherly","raw_affiliation_strings":["Department of Marketing, Oklahoma State University Stillwater, Stillwater, Oklahoma, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Marketing, Oklahoma State University Stillwater, Stillwater, Oklahoma, USA","institution_ids":["https://openalex.org/I115475287"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5026486449"],"corresponding_institution_ids":["https://openalex.org/I115475287"],"apc_list":null,"apc_paid":null,"fwci":8.7042,"has_fulltext":false,"cited_by_count":92,"citation_normalized_percentile":{"value":0.97734799,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":92,"max":99},"biblio":{"volume":"53","issue":"2","first_page":"195","last_page":"223"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9589999914169312,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9589999914169312,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.018699999898672104,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12970","display_name":"Names, Identity, and Discrimination Research","score":0.0020000000949949026,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mechanical-turk","display_name":"Mechanical Turk","score":0.7730000019073486},{"id":"https://openalex.org/keywords/online-reputation-systems","display_name":"online reputation systems","score":0.6549999713897705},{"id":"https://openalex.org/keywords/positivity-bias","display_name":"positivity bias","score":0.9800000190734863},{"id":"https://openalex.org/keywords/worker-evaluation","display_name":"worker evaluation","score":0.5350000262260437},{"id":"https://openalex.org/keywords/data-quality","display_name":"data quality","score":0.675000011920929},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"crowdsourcing","score":0.7279999852180481},{"id":"https://openalex.org/keywords/approval-rates","display_name":"approval rates","score":0.593999981880188},{"id":"https://openalex.org/keywords/status-markers","display_name":"status markers","score":0.6570000052452087}],"concepts":[{"id":"https://openalex.org/C48798503","wikidata":"https://www.wikidata.org/wiki/Q877546","display_name":"Reputation","level":2,"score":0.917237401008606},{"id":"https://openalex.org/C2776950860","wikidata":"https://www.wikidata.org/wiki/Q2914681","display_name":"Originality","level":3,"score":0.7376842498779297},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6810619831085205},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.5184421539306641},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.517709493637085},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.46930861473083496},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.42567169666290283},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3313225507736206},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.2849392294883728},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.14517995715141296},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07458147406578064},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C11012388","wikidata":"https://www.wikidata.org/wiki/Q170658","display_name":"Creativity","level":2,"score":0.0},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1108/ejm-07-2017-0491","doi":"https://doi.org/10.1108/ejm-07-2017-0491","is_oa":false,"landing_page_url":"https://doi.org/10.1108/ejm-07-2017-0491","pdf_url":null,"source":{"id":"https://openalex.org/S89389284","display_name":"European Journal of Marketing","issn_l":"0309-0566","issn":["0309-0566","1758-7123"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a-star","anvur-class-a","anvur-scientific","cwts-core","erih-plus","fnege-2","jufo-2","norway-1","poland-70","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319811","host_organization_name":"Emerald Publishing Limited","host_organization_lineage":["https://openalex.org/P4310319811"],"host_organization_lineage_names":["Emerald Publishing Limited"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"European Journal of Marketing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":95,"referenced_works":["https://openalex.org/W68991936","https://openalex.org/W640508513","https://openalex.org/W1449692765","https://openalex.org/W1488837022","https://openalex.org/W1721870969","https://openalex.org/W1872515333","https://openalex.org/W1948934543","https://openalex.org/W1973200191","https://openalex.org/W1973628995","https://openalex.org/W1975450585","https://openalex.org/W1976633387","https://openalex.org/W1984022436","https://openalex.org/W1990406041","https://openalex.org/W1993607151","https://openalex.org/W2007252106","https://openalex.org/W2009953698","https://openalex.org/W2012964103","https://openalex.org/W2022550861","https://openalex.org/W2023224020","https://openalex.org/W2023587011","https://openalex.org/W2024166834","https://openalex.org/W2037557484","https://openalex.org/W2038496767","https://openalex.org/W2042962409","https://openalex.org/W2054788835","https://openalex.org/W2064062304","https://openalex.org/W2066649540","https://openalex.org/W2070179482","https://openalex.org/W2076387137","https://openalex.org/W2078030625","https://openalex.org/W2079761107","https://openalex.org/W2080211902","https://openalex.org/W2085958018","https://openalex.org/W2086676398","https://openalex.org/W2090513801","https://openalex.org/W2103001613","https://openalex.org/W2105753174","https://openalex.org/W2106568252","https://openalex.org/W2114068176","https://openalex.org/W2115316459","https://openalex.org/W2116323796","https://openalex.org/W2116676613","https://openalex.org/W2119298903","https://openalex.org/W2123295723","https://openalex.org/W2125326971","https://openalex.org/W2126302787","https://openalex.org/W2126493218","https://openalex.org/W2128168853","https://openalex.org/W2129940040","https://openalex.org/W2131691843","https://openalex.org/W2133661776","https://openalex.org/W2134299926","https://openalex.org/W2135189989","https://openalex.org/W2135294639","https://openalex.org/W2135927844","https://openalex.org/W2141708418","https://openalex.org/W2147603330","https://openalex.org/W2150901716","https://openalex.org/W2151332190","https://openalex.org/W2156109180","https://openalex.org/W2159393007","https://openalex.org/W2160475149","https://openalex.org/W2166140580","https://openalex.org/W2170891295","https://openalex.org/W2199108421","https://openalex.org/W2223331072","https://openalex.org/W2285144430","https://openalex.org/W2304879570","https://openalex.org/W2316830863","https://openalex.org/W2320606554","https://openalex.org/W2338705265","https://openalex.org/W2395622121","https://openalex.org/W2406423356","https://openalex.org/W2489895582","https://openalex.org/W2518162742","https://openalex.org/W2567508531","https://openalex.org/W2581057531","https://openalex.org/W2585292421","https://openalex.org/W2600849123","https://openalex.org/W2603418839","https://openalex.org/W2605807262","https://openalex.org/W2609226177","https://openalex.org/W2614237344","https://openalex.org/W2625865465","https://openalex.org/W2735460460","https://openalex.org/W2811371717","https://openalex.org/W2994830416","https://openalex.org/W3121424256","https://openalex.org/W3122347011","https://openalex.org/W3123091093","https://openalex.org/W3124878131","https://openalex.org/W3125596464","https://openalex.org/W3125993287","https://openalex.org/W3141897683","https://openalex.org/W7042336540"],"related_works":["https://openalex.org/W1872515333","https://openalex.org/W2772178154","https://openalex.org/W4412438406","https://openalex.org/W2291627366","https://openalex.org/W569447453","https://openalex.org/W2613519278","https://openalex.org/W2406315853","https://openalex.org/W7214512617","https://openalex.org/W1998351012","https://openalex.org/W2766537524"],"abstract_inverted_index":{"Purpose":[0],"The":[1,35,86,196],"purpose":[2],"of":[3,12,29,37,44,80,91,97,109,205,217,227],"this":[4,38,148],"paper":[5],"is":[6,20,33],"to":[7,53,76,135,160,189,223],"investigate":[8,77],"how":[9,210],"the":[10,27,30,42,45,78,81,89,107,110,124,144,168,180,201,215,225],"effectiveness":[11],"systems":[13,207],"for":[14],"ensuring":[15],"cooperation":[16],"in":[17,26,72,95,128,140,147,153,162,221],"online":[18],"transactions":[19],"impacted":[21],"by":[22,114],"a":[23,92],"positivity":[24,93],"bias":[25,39,94],"evaluation":[28],"work":[31],"that":[32,129,171],"produced.":[34],"presence":[36,90],"can":[40,213],"reduce":[41,224],"informativeness":[43,79],"reputation":[46,83,111,206],"system":[47,112],"and":[48,61,113,203,208],"negatively":[49],"impact":[50],"its":[51],"ability":[52],"ensure":[54],"quality.":[55,163],"Design/methodology/approach":[56],"This":[57],"research":[58,149,197],"combines":[59],"survey":[60],"experimental":[62],"methods,":[63],"collecting":[64],"data":[65],"from":[66],"1,875":[67],"Mechanical":[68],"Turk":[69],"(MTurk)":[70],"workers":[71,98,121,192],"five":[73],"studies":[74],"designed":[75],"MTurk":[82,120],"system.":[84],"Findings":[85],"findings":[87,125],"demonstrate":[88],"evaluations":[96,222],"on":[99,193],"MTurk,":[100],"which":[101,155],"leaves":[102],"them":[103],"undifferentiated,":[104],"except":[105],"at":[106],"extremity":[108],"status":[115],"markers.":[116],"Research":[117],"limitations/implications":[118],"Because":[119],"self-select":[122],"tasks,":[123],"are":[126,138,150],"limited":[127],"they":[130],"may":[131,156],"only":[132],"be":[133,187],"generalizable":[134],"those":[136],"who":[137],"interested":[139],"research-related":[141],"work.":[142],"Further,":[143],"tasks":[145],"used":[146,188],"largely":[151],"subjective":[152],"nature,":[154],"decrease":[157],"their":[158],"sensitivity":[159],"differences":[161],"Practical":[164],"implications":[165],"For":[166],"researchers,":[167],"results":[169],"suggest":[170],"requiring":[172],"99":[173],"per":[174,184],"cent":[175],"approval":[176],"rates":[177],"(rather":[178],"than":[179],"previously":[181],"advised":[182],"95":[183],"cent)":[185],"should":[186],"identify":[190],"high-quality":[191],"MTurk.":[194],"Originality/value":[195],"provides":[198],"insights":[199],"into":[200],"design":[202,211],"use":[204],"demonstrates":[209],"decisions":[212],"exacerbate":[214],"effect":[216],"naturally":[218],"occurring":[219],"biases":[220],"utility":[226],"these":[228],"systems.":[229]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2908862202","counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":13},{"year":2023,"cited_by_count":15},{"year":2022,"cited_by_count":13},{"year":2021,"cited_by_count":16},{"year":2020,"cited_by_count":17},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":2}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2019-01-25T00:00:00"}
