{"id":"https://openalex.org/W4367046989","doi":"https://doi.org/10.1145/3578503.3583622","title":"Who Broke Amazon Mechanical Turk?","display_name":"Who Broke Amazon Mechanical Turk?","publication_year":2023,"publication_date":"2023-04-26","ids":{"openalex":"https://openalex.org/W4367046989","doi":"https://doi.org/10.1145/3578503.3583622"},"language":"en","primary_location":{"id":"doi:10.1145/3578503.3583622","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3578503.3583622","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th ACM Web Science Conference 2023","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103218755","display_name":"Catherine C. Marshall","orcid":"https://orcid.org/0009-0001-2635-2915"},"institutions":[{"id":"https://openalex.org/I91045830","display_name":"Texas A&M University","ror":"https://ror.org/01f5ytq51","country_code":"US","type":"education","lineage":["https://openalex.org/I91045830"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Catherine C. Marshall","raw_affiliation_strings":["Department of Computer Science and Engineering, Texas A and M University, USA"],"raw_orcid":"https://orcid.org/0009-0001-2635-2915","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Texas A and M University, USA","institution_ids":["https://openalex.org/I91045830"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012829856","display_name":"Partha S.R. Goguladinne","orcid":"https://orcid.org/0009-0000-3472-4371"},"institutions":[{"id":"https://openalex.org/I91045830","display_name":"Texas A&M University","ror":"https://ror.org/01f5ytq51","country_code":"US","type":"education","lineage":["https://openalex.org/I91045830"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Partha S.R. Goguladinne","raw_affiliation_strings":["Department of Computer Science and Engineering, Texas A&M University, USA"],"raw_orcid":"https://orcid.org/0009-0000-3472-4371","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Texas A&M University, USA","institution_ids":["https://openalex.org/I91045830"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043446043","display_name":"Mudit Maheshwari","orcid":"https://orcid.org/0009-0007-9768-5729"},"institutions":[{"id":"https://openalex.org/I91045830","display_name":"Texas A&M University","ror":"https://ror.org/01f5ytq51","country_code":"US","type":"education","lineage":["https://openalex.org/I91045830"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mudit Maheshwari","raw_affiliation_strings":["Department of Computer Science and Engineering, Texas A&M University, USA"],"raw_orcid":"https://orcid.org/0009-0007-9768-5729","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Texas A&M University, USA","institution_ids":["https://openalex.org/I91045830"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039072801","display_name":"Apoorva Sathe","orcid":"https://orcid.org/0009-0006-9797-4224"},"institutions":[{"id":"https://openalex.org/I91045830","display_name":"Texas A&M University","ror":"https://ror.org/01f5ytq51","country_code":"US","type":"education","lineage":["https://openalex.org/I91045830"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Apoorva Sathe","raw_affiliation_strings":["Department of Computer Science and Engineering, Texas A&M University, USA"],"raw_orcid":"https://orcid.org/0009-0006-9797-4224","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Texas A&M University, USA","institution_ids":["https://openalex.org/I91045830"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077860280","display_name":"Frank Shipman","orcid":"https://orcid.org/0009-0007-1469-9835"},"institutions":[{"id":"https://openalex.org/I91045830","display_name":"Texas A&M University","ror":"https://ror.org/01f5ytq51","country_code":"US","type":"education","lineage":["https://openalex.org/I91045830"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Frank M. Shipman","raw_affiliation_strings":["Department of Computer Science and Engineering, Texas A&M University, USA"],"raw_orcid":"https://orcid.org/0009-0007-1469-9835","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Texas A&M University, USA","institution_ids":["https://openalex.org/I91045830"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I91045830"],"apc_list":null,"apc_paid":null,"fwci":5.917,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.97544338,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"335","last_page":"345"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11045","display_name":"Privacy, Security, and Data Protection","score":0.942799985408783,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/fluency","display_name":"Fluency","score":0.7429106831550598},{"id":"https://openalex.org/keywords/likert-scale","display_name":"Likert scale","score":0.5738652348518372},{"id":"https://openalex.org/keywords/survey-data-collection","display_name":"Survey data collection","score":0.5234978795051575},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5185380578041077},{"id":"https://openalex.org/keywords/qualitative-property","display_name":"Qualitative property","score":0.4702272117137909},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.464346706867218},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4113911986351013},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.37119561433792114},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.15829044580459595},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.15716159343719482},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.13297763466835022},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.11972838640213013},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.115592360496521},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.08175322413444519}],"concepts":[{"id":"https://openalex.org/C2777413886","wikidata":"https://www.wikidata.org/wiki/Q3276013","display_name":"Fluency","level":2,"score":0.7429106831550598},{"id":"https://openalex.org/C105776082","wikidata":"https://www.wikidata.org/wiki/Q617473","display_name":"Likert scale","level":2,"score":0.5738652348518372},{"id":"https://openalex.org/C198477413","wikidata":"https://www.wikidata.org/wiki/Q7647069","display_name":"Survey data collection","level":2,"score":0.5234978795051575},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5185380578041077},{"id":"https://openalex.org/C87156501","wikidata":"https://www.wikidata.org/wiki/Q7268708","display_name":"Qualitative property","level":2,"score":0.4702272117137909},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.464346706867218},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4113911986351013},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.37119561433792114},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.15829044580459595},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.15716159343719482},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.13297763466835022},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.11972838640213013},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.115592360496521},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.08175322413444519},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3578503.3583622","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3578503.3583622","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th ACM Web Science Conference 2023","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W2028511812","https://openalex.org/W2048601276","https://openalex.org/W2076387137","https://openalex.org/W2105753174","https://openalex.org/W2114269021","https://openalex.org/W2116323796","https://openalex.org/W2150597317","https://openalex.org/W2293765620","https://openalex.org/W2331462388","https://openalex.org/W2607669503","https://openalex.org/W2610617162","https://openalex.org/W2726137762","https://openalex.org/W2756079620","https://openalex.org/W2783946743","https://openalex.org/W2789412903","https://openalex.org/W2811176066","https://openalex.org/W2811371717","https://openalex.org/W2941417947","https://openalex.org/W2979864122","https://openalex.org/W2988983801","https://openalex.org/W3011179284","https://openalex.org/W3108748085","https://openalex.org/W3115100653","https://openalex.org/W3123404044","https://openalex.org/W3125334609","https://openalex.org/W3202749896","https://openalex.org/W3215436086","https://openalex.org/W4294903874","https://openalex.org/W4366588458"],"related_works":["https://openalex.org/W2365169615","https://openalex.org/W1970538215","https://openalex.org/W2400151637","https://openalex.org/W2975827637","https://openalex.org/W2354089692","https://openalex.org/W595497825","https://openalex.org/W4236323843","https://openalex.org/W3123142427","https://openalex.org/W3009841415","https://openalex.org/W4296737668"],"abstract_inverted_index":{"We":[0,25,127],"present":[1],"the":[2,36,81,132,135,146],"results":[3],"of":[4,10,52,98,105,134],"a":[5,13,65,96],"survey":[6,31,38,165],"fielded":[7,39],"in":[8,43,69,74,89],"June":[9],"2022":[11],"as":[12],"lens":[14],"to":[15,71,86,95,108,113,130,154],"examine":[16],"recent":[17],"data":[18,28,34,55,61,82,143],"reliability":[19,83],"issues":[20],"on":[21,129,140,145],"Amazon":[22],"Mechanical":[23],"Turk.":[24],"contrast":[26],"bad":[27,33,90,124],"from":[29,35,64],"this":[30],"with":[32],"same":[37],"among":[40],"US":[41],"workers":[42],"October":[44],"2013,":[45],"April":[46],"2018,":[47],"and":[48,123,144],"February":[49],"2019.":[50],"Application":[51],"an":[53,87],"established":[54],"cleaning":[56],"scheme":[57],"reveals":[58],"that":[59,167],"unusable":[60],"has":[62],"risen":[63],"little":[66],"over":[67],"2%":[68],"2013":[70],"almost":[72],"90%":[73],"2022.":[75],"Through":[76],"symptomatic":[77],"diagnosis,":[78],"we":[79],"attribute":[80],"drop":[84],"not":[85],"increase":[88],"faith":[91,125],"work,":[92],"but":[93],"rather":[94],"continuum":[97],"English":[99],"proficiency":[100],"levels.":[101],"A":[102],"qualitative":[103,148],"analysis":[104],"workers\u2019":[106],"responses":[107,166],"open-ended":[109],"questions":[110],"allows":[111],"us":[112],"distinguish":[114],"between":[115],"low":[116,137],"fluency":[117,120,138],"workers,":[118,121],"ultra-low":[119],"satisficers,":[122],"workers.":[126],"go":[128],"show":[131],"effects":[133],"new":[136],"work":[139],"Likert":[141],"scale":[142],"study's":[147],"results.":[149],"Attention":[150],"checks":[151],"are":[152],"shown":[153],"be":[155,169],"much":[156],"less":[157],"effective":[158],"than":[159],"they":[160],"once":[161],"were":[162],"at":[163],"identifying":[164],"should":[168],"discarded.":[170]},"counts_by_year":[{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":4}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}