{"id":"https://openalex.org/W3208472527","doi":"https://doi.org/10.1145/3462204.3481729","title":"Investigating and Mitigating Biases in Crowdsourced Data","display_name":"Investigating and Mitigating Biases in Crowdsourced Data","publication_year":2021,"publication_date":"2021-10-22","ids":{"openalex":"https://openalex.org/W3208472527","doi":"https://doi.org/10.1145/3462204.3481729","mag":"3208472527"},"language":"en","primary_location":{"id":"doi:10.1145/3462204.3481729","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3462204.3481729","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Publication of the 2021 Conference on Computer Supported Cooperative Work and Social Computing","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014686840","display_name":"Danula Hettiachchi","orcid":"https://orcid.org/0000-0003-3875-5727"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Danula Hettiachchi","raw_affiliation_strings":["RMIT University, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072614956","display_name":"Mark Sanderson","orcid":"https://orcid.org/0000-0003-0487-9609"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Mark Sanderson","raw_affiliation_strings":["RMIT University, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023644163","display_name":"Jorge Gon\u00e7alves","orcid":"https://orcid.org/0000-0002-0117-0322"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"The University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Jorge Goncalves","raw_affiliation_strings":["University of Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"University of Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067689431","display_name":"Simo Hosio","orcid":"https://orcid.org/0000-0002-9609-0965"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Simo Hosio","raw_affiliation_strings":["University of Oulu, Finland"],"affiliations":[{"raw_affiliation_string":"University of Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078735102","display_name":"Gabriella Kazai","orcid":"https://orcid.org/0009-0002-5158-6630"},"institutions":[{"id":"https://openalex.org/I4210108625","display_name":"Microsoft (United Kingdom)","ror":"https://ror.org/01rw27z95","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210108625"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Gabriella Kazai","raw_affiliation_strings":["Microsoft, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Microsoft, United Kingdom","institution_ids":["https://openalex.org/I4210108625"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023397430","display_name":"Matthew Lease","orcid":"https://orcid.org/0000-0002-0056-2834"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Lease","raw_affiliation_strings":["University of Texas at Austin and Amazon, USA"],"affiliations":[{"raw_affiliation_string":"University of Texas at Austin and Amazon, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025626087","display_name":"Mike Schaekermann","orcid":"https://orcid.org/0000-0002-1735-9680"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mike Schaekermann","raw_affiliation_strings":["Amazon, Canada"],"affiliations":[{"raw_affiliation_string":"Amazon, Canada","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076265623","display_name":"Emine Y\u0131lmaz","orcid":"https://orcid.org/0000-0002-3434-8932"},"institutions":[{"id":"https://openalex.org/I4210123934","display_name":"Amazon (United Kingdom)","ror":"https://ror.org/02xey9634","country_code":"GB","type":"company","lineage":["https://openalex.org/I1311688040","https://openalex.org/I4210123934"]},{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Emine Yilmaz","raw_affiliation_strings":["University College London and Amazon, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University College London and Amazon, United Kingdom","institution_ids":["https://openalex.org/I4210123934","https://openalex.org/I45129253"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5014686840"],"corresponding_institution_ids":["https://openalex.org/I82951845"],"apc_list":null,"apc_paid":null,"fwci":2.1909,"has_fulltext":false,"cited_by_count":16,"citation_normalized_percentile":{"value":0.89510672,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"331","last_page":"334"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9911999702453613,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9793999791145325,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9680904150009155},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7250880002975464},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6857873201370239},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6577651500701904},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5985831022262573},{"id":"https://openalex.org/keywords/citizen-science","display_name":"Citizen science","score":0.5453254580497742},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4942157566547394},{"id":"https://openalex.org/keywords/plan","display_name":"Plan (archaeology)","score":0.43804439902305603},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.42683690786361694},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.32304298877716064},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.12517204880714417},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.07975757122039795}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9680904150009155},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7250880002975464},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6857873201370239},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6577651500701904},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5985831022262573},{"id":"https://openalex.org/C197352329","wikidata":"https://www.wikidata.org/wiki/Q1093434","display_name":"Citizen science","level":2,"score":0.5453254580497742},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4942157566547394},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.43804439902305603},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.42683690786361694},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32304298877716064},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.12517204880714417},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.07975757122039795},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3462204.3481729","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3462204.3481729","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Publication of the 2021 Conference on Computer Supported Cooperative Work and Social Computing","raw_type":"proceedings-article"},{"id":"pmh:oai:figshare.com:article/27592374","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:figshare.com:article/27592374","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":43,"referenced_works":["https://openalex.org/W320171562","https://openalex.org/W1927004789","https://openalex.org/W1987831316","https://openalex.org/W1994322930","https://openalex.org/W2000727220","https://openalex.org/W2051752522","https://openalex.org/W2053596265","https://openalex.org/W2116984840","https://openalex.org/W2161709257","https://openalex.org/W2180771322","https://openalex.org/W2262972580","https://openalex.org/W2293340641","https://openalex.org/W2563852449","https://openalex.org/W2606013986","https://openalex.org/W2613995098","https://openalex.org/W2754913727","https://openalex.org/W2783668259","https://openalex.org/W2788241675","https://openalex.org/W2896331720","https://openalex.org/W2899134503","https://openalex.org/W2941486823","https://openalex.org/W2941985495","https://openalex.org/W2942604226","https://openalex.org/W2949678053","https://openalex.org/W2952887699","https://openalex.org/W2962707256","https://openalex.org/W2963174898","https://openalex.org/W2964237876","https://openalex.org/W2970442950","https://openalex.org/W2986466127","https://openalex.org/W2988571156","https://openalex.org/W2999380901","https://openalex.org/W2999697822","https://openalex.org/W3009312460","https://openalex.org/W3012819905","https://openalex.org/W3014880912","https://openalex.org/W3092018334","https://openalex.org/W3093582825","https://openalex.org/W3121257585","https://openalex.org/W3152757293","https://openalex.org/W3160735296","https://openalex.org/W3181414820","https://openalex.org/W3205934028"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W2555391182"],"abstract_inverted_index":{"It":[0,18],"is":[1,19],"common":[2],"practice":[3],"for":[4,14,44,243],"machine":[5],"learning":[6],"systems":[7],"to":[8,56,65,95,105,117,122,158,165,202,209,216,225,232,240],"rely":[9],"on":[10,108],"crowdsourced":[11,114,220],"label":[12,27,72],"data":[13,28,47,83,88,115],"training":[15],"and":[16,67,86,116,125,130,140,154,167,176,184,213],"evaluation.":[17],"also":[20,223],"well-known":[21],"that":[22,136],"biases":[23,31,69,80,112,127,159,169,218],"present":[24],"in":[25,32,113,160,219],"the":[26,33,41,53,58,71,82,132,161,173,185,191,233],"can":[29,143],"induce":[30],"trained":[34],"models.":[35],"Biases":[36],"may":[37,181],"be":[38,49,91,238],"introduced":[39],"by":[40,52],"mechanisms":[42,54],"used":[43],"deciding":[45],"what":[46],"should/could":[48],"labelled":[50,162],"or":[51,204],"employed":[55],"obtain":[57],"labels.":[59],"Various":[60],"approaches":[61,180],"have":[62],"been":[63,75],"proposed":[64],"detect":[66],"correct":[68],"once":[70],"dataset":[73,246],"has":[74],"constructed.":[76],"However,":[77],"proactively":[78],"reducing":[79],"during":[81,129],"labelling":[84,133,174],"phase":[85],"ensuring":[87],"fairness":[89],"could":[90],"more":[92],"economical":[93],"compared":[94],"post-processing":[96],"bias":[97],"mitigation":[98,179],"approaches.":[99],"In":[100],"this":[101],"workshop,":[102,234],"we":[103],"aim":[104],"foster":[106],"discussion":[107],"ongoing":[109],"research":[110,120,200],"around":[111],"identify":[118],"future":[119],"directions":[121],"detect,":[123],"quantify":[124,166],"mitigate":[126,168],"before,":[128],"after":[131],"process":[134],"such":[135,178],"both":[137],"task":[138],"requesters":[139],"crowd":[141],"workers":[142,183],"benefit.":[144],"We":[145,222],"will":[146,193,237],"explore":[147],"how":[148,164,177],"specific":[149],"crowdsourcing":[150,186,210],"workflows,":[151],"worker":[152],"attributes,":[153],"work":[155,214],"practices":[156,215],"contribute":[157],"data;":[163],"as":[170],"part":[171],"of":[172,190,198],"process;":[175],"impact":[182],"ecosystem.":[187],"The":[188],"outcome":[189],"workshop":[192],"include":[194],"a":[195,199,227,244],"collaborative":[196],"publication":[197],"agenda":[201],"improve":[203],"develop":[205],"novel":[206],"methods":[207],"relating":[208],"tools,":[211],"processes":[212],"address":[217],"data.":[221],"plan":[224],"run":[226],"Crowd":[228],"Bias":[229],"Challenge":[230],"prior":[231],"where":[235],"participants":[236],"asked":[239],"collect":[241],"labels":[242],"given":[245],"while":[247],"minimising":[248],"potential":[249],"biases.":[250]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":1}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}