{"id":"https://openalex.org/W4390860179","doi":"https://doi.org/10.1145/3639452","title":"Resolving the Human-subjects Status of Machine Learning's Crowdworkers","display_name":"Resolving the Human-subjects Status of Machine Learning's Crowdworkers","publication_year":2023,"publication_date":"2023-12-31","ids":{"openalex":"https://openalex.org/W4390860179","doi":"https://doi.org/10.1145/3639452"},"language":"en","primary_location":{"id":"doi:10.1145/3639452","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639452","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639452","source":{"id":"https://openalex.org/S45584542","display_name":"Queue","issn_l":"1542-7730","issn":["1542-7730","1542-7749"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Queue","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3639452","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056609277","display_name":"Divyansh Kaushik","orcid":null},"institutions":[{"id":"https://openalex.org/I34055209","display_name":"Federation of American Scientists","ror":"https://ror.org/00m9r8n64","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I34055209"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Divyansh Kaushik","raw_affiliation_strings":["Federation of American Scientists"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federation of American Scientists","institution_ids":["https://openalex.org/I34055209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029448258","display_name":"Zachary C. Lipton","orcid":"https://orcid.org/0000-0002-3824-4241"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zachary C. Lipton","raw_affiliation_strings":["CMU"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CMU","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5042431726","display_name":"Alex John London","orcid":"https://orcid.org/0000-0002-6450-0309"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alex John London","raw_affiliation_strings":["CMU"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CMU","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5056609277"],"corresponding_institution_ids":["https://openalex.org/I34055209"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.21017638,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"21","issue":"6","first_page":"101","last_page":"127"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9937999844551086,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8175413608551025},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6110057830810547},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.5847049951553345},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5630213618278503},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5050018429756165},{"id":"https://openalex.org/keywords/diversity","display_name":"Diversity (politics)","score":0.49155983328819275},{"id":"https://openalex.org/keywords/common-rule","display_name":"Common Rule","score":0.48755091428756714},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.33944880962371826},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2517966032028198},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.22886702418327332},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.10650339722633362}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8175413608551025},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6110057830810547},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.5847049951553345},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5630213618278503},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5050018429756165},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.49155983328819275},{"id":"https://openalex.org/C2781403167","wikidata":"https://www.wikidata.org/wiki/Q5153464","display_name":"Common Rule","level":4,"score":0.48755091428756714},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.33944880962371826},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2517966032028198},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.22886702418327332},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.10650339722633362},{"id":"https://openalex.org/C204787440","wikidata":"https://www.wikidata.org/wiki/Q188504","display_name":"Alternative medicine","level":2,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C68122502","wikidata":"https://www.wikidata.org/wiki/Q764527","display_name":"Informed consent","level":3,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3639452","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639452","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639452","source":{"id":"https://openalex.org/S45584542","display_name":"Queue","issn_l":"1542-7730","issn":["1542-7730","1542-7749"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Queue","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3639452","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639452","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639452","source":{"id":"https://openalex.org/S45584542","display_name":"Queue","issn_l":"1542-7730","issn":["1542-7730","1542-7749"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Queue","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320310174","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305"},{"id":"https://openalex.org/F4320310207","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33"},{"id":"https://openalex.org/F4320332530","display_name":"Medical Center, University of Pittsburgh","ror":"https://ror.org/04ehecz88"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4390860179.pdf","grobid_xml":"https://content.openalex.org/works/W4390860179.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W1847618513","https://openalex.org/W2129494713","https://openalex.org/W2251411520","https://openalex.org/W2511234952","https://openalex.org/W2742139733","https://openalex.org/W2807760453","https://openalex.org/W2890894339","https://openalex.org/W2898695519","https://openalex.org/W2963846996","https://openalex.org/W2969360677","https://openalex.org/W2970283086","https://openalex.org/W2970442950","https://openalex.org/W2999071598","https://openalex.org/W3035503910","https://openalex.org/W3096482855","https://openalex.org/W3098998028","https://openalex.org/W3100420479","https://openalex.org/W3103873238","https://openalex.org/W3106454043","https://openalex.org/W3136687158","https://openalex.org/W3171896931","https://openalex.org/W3173528751","https://openalex.org/W3173826007","https://openalex.org/W3207937903","https://openalex.org/W4205743601"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W3207526114","https://openalex.org/W4286908577"],"abstract_inverted_index":{"In":[0],"recent":[1],"years,":[2],"machine":[3],"learning":[4],"(ML)":[5],"has":[6],"relied":[7],"heavily":[8],"on":[9,114,134],"crowdworkers":[10,45,70],"both":[11,28,140],"for":[12,16,122,202],"building":[13],"datasets":[14],"and":[15,32,65,74,99,149,173,179,196,232],"addressing":[17],"research":[18,123,181,189,225],"questions":[19,190,207],"requiring":[20],"human":[21,53,72,81],"interaction":[22],"or":[23],"judgment.":[24],"The":[25],"diversity":[26],"of":[27,35,49,95,107,136,167,177],"the":[29,33,36,93,104,127,144,153,164,200,218],"tasks":[30],"performed":[31],"uses":[34],"resulting":[37],"data":[38,146,197,230],"render":[39],"it":[40],"difficult":[41],"to":[42,118,183,205,243],"determine":[43],"when":[44],"are":[46,57,191],"best":[47],"thought":[48],"as":[50,71],"workers":[51,168],"versus":[52],"subjects.":[54,82],"These":[55],"difficulties":[56],"compounded":[58],"by":[59,162,228],"conflicting":[60],"policies,":[61],"with":[62,97],"some":[63],"institutions":[64],"researchers":[66,222],"regarding":[67],"all":[68],"ML":[69,85,108,180],"subjects":[73],"others":[75],"holding":[76],"that":[77],"they":[78],"rarely":[79],"constitute":[80],"Notably":[83],"few":[84],"papers":[86],"involving":[87],"crowdwork":[88],"mention":[89],"IRB":[90],"oversight,":[91],"raising":[92],"prospect":[94],"non-compliance":[96],"ethical":[98],"regulatory":[100],"requirements.":[101],"We":[102,157],"investigate":[103],"appropriate":[105],"designation":[106],"crowdsourcing":[109],"studies,":[110],"focusing":[111],"our":[112],"inquiry":[113],"natural":[115],"language":[116],"processing":[117],"expose":[119],"unique":[120],"challenges":[121,160],"oversight.":[124],"Crucially,":[125],"under":[126],"U.S.":[128],"Common":[129,219],"Rule,":[130,220],"these":[131,245],"judgments":[132],"hinge":[133],"determinations":[135],"aboutness":[137],",":[138],"concerning":[139],"whom":[141,150],"(or":[142,151],"what)":[143,152],"collected":[145],"is":[147,155],"about":[148],"analysis":[154,212,233],"about.":[156],"highlight":[158],"two":[159],"posed":[161],"ML:":[163],"same":[165],"set":[166],"can":[169,223],"serve":[170],"multiple":[171],"roles":[172],"provide":[174],"many":[175],"sorts":[176],"information;":[178],"tends":[182],"embrace":[184],"a":[185,214],"dynamic":[186],"workflow,":[187],"where":[188,221],"seldom":[192],"stated":[193],"ex":[194],"ante":[195],"sharing":[198],"opens":[199],"door":[201],"future":[203],"studies":[204],"aim":[206],"at":[208],"different":[209],"targets.":[210],"Our":[211],"exposes":[213],"potential":[215],"loophole":[216],"in":[217],"elude":[224],"ethics":[226],"oversight":[227],"splitting":[229],"collection":[231],"into":[234],"distinct":[235],"studies.":[236],"Finally,":[237],"we":[238],"offer":[239],"several":[240],"policy":[241],"recommendations":[242],"address":[244],"concerns.":[246]},"counts_by_year":[],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}