{"id":"https://openalex.org/W3108534331","doi":"https://doi.org/10.1145/3428029.3428062","title":"Crowdsourcing in Computing Education Research: Case\u00a0Amazon\u00a0MTurk","display_name":"Crowdsourcing in Computing Education Research: Case\u00a0Amazon\u00a0MTurk","publication_year":2020,"publication_date":"2020-11-19","ids":{"openalex":"https://openalex.org/W3108534331","doi":"https://doi.org/10.1145/3428029.3428062","mag":"3108534331"},"language":"en","primary_location":{"id":"doi:10.1145/3428029.3428062","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3428029.3428062","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Koli Calling '20: Proceedings of the 20th Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://aaltodoc.aalto.fi/handle/123456789/111960","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076828114","display_name":"Arto Hellas","orcid":"https://orcid.org/0000-0001-6502-209X"},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":true,"raw_author_name":"Arto Hellas","raw_affiliation_strings":["Aalto University, Finland"],"affiliations":[{"raw_affiliation_string":"Aalto University, Finland","institution_ids":["https://openalex.org/I9927081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076312356","display_name":"Albina Zavgorodniaia","orcid":null},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Albina Zavgorodniaia","raw_affiliation_strings":["Aalto University, Finland"],"affiliations":[{"raw_affiliation_string":"Aalto University, Finland","institution_ids":["https://openalex.org/I9927081"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5060225735","display_name":"Juha Sorva","orcid":"https://orcid.org/0009-0003-1727-1317"},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Juha Sorva","raw_affiliation_strings":["Aalto University, Finland"],"affiliations":[{"raw_affiliation_string":"Aalto University, Finland","institution_ids":["https://openalex.org/I9927081"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5076828114"],"corresponding_institution_ids":["https://openalex.org/I9927081"],"apc_list":null,"apc_paid":null,"fwci":2.2963,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.9032051,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9886999726295471,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.990666925907135},{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"Generalizability theory","score":0.7481703162193298},{"id":"https://openalex.org/keywords/amazon-rainforest","display_name":"Amazon rainforest","score":0.6300842761993408},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.591743528842926},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5909513831138611},{"id":"https://openalex.org/keywords/crowdsourcing-software-development","display_name":"Crowdsourcing software development","score":0.48822978138923645},{"id":"https://openalex.org/keywords/workforce","display_name":"Workforce","score":0.46707776188850403},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.44081878662109375},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.34331315755844116},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.22738859057426453},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.10643652081489563}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.990666925907135},{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.7481703162193298},{"id":"https://openalex.org/C535291247","wikidata":"https://www.wikidata.org/wiki/Q177567","display_name":"Amazon rainforest","level":2,"score":0.6300842761993408},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.591743528842926},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5909513831138611},{"id":"https://openalex.org/C32810761","wikidata":"https://www.wikidata.org/wiki/Q17082146","display_name":"Crowdsourcing software development","level":5,"score":0.48822978138923645},{"id":"https://openalex.org/C2778139618","wikidata":"https://www.wikidata.org/wiki/Q13440398","display_name":"Workforce","level":2,"score":0.46707776188850403},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.44081878662109375},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.34331315755844116},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.22738859057426453},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.10643652081489563},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.0},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.0},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3428029.3428062","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3428029.3428062","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Koli Calling '20: Proceedings of the 20th Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"},{"id":"pmh:oai:aaltodoc.aalto.fi:123456789/111960","is_oa":true,"landing_page_url":"https://aaltodoc.aalto.fi/handle/123456789/111960","pdf_url":null,"source":{"id":"https://openalex.org/S4306401662","display_name":"Aaltodoc (Aalto University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I9927081","host_organization_name":"Aalto University","host_organization_lineage":["https://openalex.org/I9927081"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"acceptedVersion"}],"best_oa_location":{"id":"pmh:oai:aaltodoc.aalto.fi:123456789/111960","is_oa":true,"landing_page_url":"https://aaltodoc.aalto.fi/handle/123456789/111960","pdf_url":null,"source":{"id":"https://openalex.org/S4306401662","display_name":"Aaltodoc (Aalto University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I9927081","host_organization_name":"Aalto University","host_organization_lineage":["https://openalex.org/I9927081"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"acceptedVersion"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W1975464614","https://openalex.org/W2057014209","https://openalex.org/W2065762074","https://openalex.org/W2079172301","https://openalex.org/W2097107772","https://openalex.org/W2146496290","https://openalex.org/W2171913066","https://openalex.org/W2207738593","https://openalex.org/W2606994258","https://openalex.org/W2612388249","https://openalex.org/W2748486817","https://openalex.org/W2763777702","https://openalex.org/W2767914643","https://openalex.org/W2809393522","https://openalex.org/W2900485379","https://openalex.org/W2905172027","https://openalex.org/W2912623520","https://openalex.org/W2941158663","https://openalex.org/W2963562081","https://openalex.org/W2966150097","https://openalex.org/W2973748760","https://openalex.org/W2979864122","https://openalex.org/W3031792761","https://openalex.org/W3032903422","https://openalex.org/W3081112910","https://openalex.org/W3123404044","https://openalex.org/W4229581335","https://openalex.org/W4234229076","https://openalex.org/W4234844921","https://openalex.org/W4365799931"],"related_works":["https://openalex.org/W2512762878","https://openalex.org/W162737300","https://openalex.org/W2520748299","https://openalex.org/W2033745466","https://openalex.org/W2520160102","https://openalex.org/W2046555014","https://openalex.org/W3131060711","https://openalex.org/W2548342535","https://openalex.org/W3125008079","https://openalex.org/W2287983667"],"abstract_inverted_index":{"Crowdsourcing":[0],"platforms":[1],"such":[2],"as":[3,61],"Amazon":[4,46],"MTurk":[5,47],"provide":[6],"access":[7],"to":[8,17,129,139],"a":[9,21,49],"human":[10],"workforce":[11],"that":[12,34,80,120,137],"can":[13,134,150],"be":[14,151],"given":[15],"tasks":[16],"complete":[18],"online":[19],"for":[20,48,90],"fee.":[22],"In":[23,98],"this":[24],"article,":[25],"we":[26,38,81,87,102],"review":[27],"studies":[28],"in":[29,55,66,92],"computing":[30],"education":[31],"research":[32,70,144],"(CER)":[33],"rely":[35],"on":[36],"crowdsourcing;":[37],"also":[39],"describe":[40],"our":[41,85,99,114],"own":[42],"experiences":[43,115],"of":[44,158],"using":[45],"CER":[50,93,122],"study.":[51],"We":[52],"discuss":[53],"challenges":[54],"recruiting":[56],"workers":[57,111],"with":[58,77],"specific":[59,140],"backgrounds\u2014such":[60],"no":[62],"programming":[63],"experience\u2014and":[64],"considerations":[65],"filtering":[67],"out":[68],"unreliable":[69],"participants.":[71],"Combining":[72],"recommendations":[73],"from":[74],"the":[75,78,117,154],"literature":[76,118],"lessons":[79],"learned":[82],"whilst":[83],"conducting":[84],"study,":[86,101],"synthesize":[88],"advice":[89],"researchers":[91],"who":[94],"are":[95],"considering":[96],"crowdsourcing.":[97],"case":[100],"did":[103],"not":[104],"find":[105],"widespread":[106],"foul":[107],"play":[108],"by":[109],"crowdsourced":[110,121,132,159],"and,":[112],"overall,":[113],"and":[116,156],"suggest":[119],"is":[123,145],"feasible.":[124],"It":[125],"is,":[126],"however,":[127],"uncertain":[128],"what":[130],"extent":[131],"data":[133],"produce":[135],"answers":[136],"apply":[138],"educational":[141],"contexts.":[142],"More":[143],"needed":[146],"before":[147],"definitive":[148],"conclusions":[149],"drawn":[152],"about":[153],"validity":[155],"generalizability":[157],"CER.":[160]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1}],"updated_date":"2026-04-01T17:29:45.350535","created_date":"2025-10-10T00:00:00"}