{"id":"https://openalex.org/W2945048417","doi":"https://doi.org/10.1145/3308558.3320096","title":"Crowdsourcing Inclusivity: Dealing with Diversity of Opinions, Perspectives and Ambiguity in Annotated Data","display_name":"Crowdsourcing Inclusivity: Dealing with Diversity of Opinions, Perspectives and Ambiguity in Annotated Data","publication_year":2019,"publication_date":"2019-05-13","ids":{"openalex":"https://openalex.org/W2945048417","doi":"https://doi.org/10.1145/3308558.3320096","mag":"2945048417"},"language":"en","primary_location":{"id":"doi:10.1145/3308558.3320096","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308558.3320096","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3308558.3320096","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of The 2019 World Wide Web Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3308558.3320096","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016381010","display_name":"Lora Aroyo","orcid":"https://orcid.org/0000-0001-9402-1133"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Lora Aroyo","raw_affiliation_strings":["Google,New York, New York, USA"],"affiliations":[{"raw_affiliation_string":"Google,New York, New York, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021638892","display_name":"Anca Dumitrache","orcid":"https://orcid.org/0000-0003-1537-1763"},"institutions":[{"id":"https://openalex.org/I865915315","display_name":"Vrije Universiteit Amsterdam","ror":"https://ror.org/008xxew50","country_code":"NL","type":"education","lineage":["https://openalex.org/I865915315"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Anca Dumitrache","raw_affiliation_strings":["Vrije Universiteit Amsterdam,Netherlands"],"affiliations":[{"raw_affiliation_string":"Vrije Universiteit Amsterdam,Netherlands","institution_ids":["https://openalex.org/I865915315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056796427","display_name":"Oana Inel","orcid":"https://orcid.org/0000-0003-4691-6586"},"institutions":[{"id":"https://openalex.org/I865915315","display_name":"Vrije Universiteit Amsterdam","ror":"https://ror.org/008xxew50","country_code":"NL","type":"education","lineage":["https://openalex.org/I865915315"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Oana Inel","raw_affiliation_strings":["Vrije Universiteit Amsterdam,Netherlands"],"affiliations":[{"raw_affiliation_string":"Vrije Universiteit Amsterdam,Netherlands","institution_ids":["https://openalex.org/I865915315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010729017","display_name":"Zolt\u00e1n Szl\u00e1vik","orcid":"https://orcid.org/0000-0002-2781-3795"},"institutions":[{"id":"https://openalex.org/I4210148065","display_name":"IBM (Netherlands)","ror":"https://ror.org/055yb9v95","country_code":"NL","type":"company","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210148065"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Zolt\u00e1n Szl\u00e1vik","raw_affiliation_strings":["IBM Benelux CAS,Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"IBM Benelux CAS,Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210148065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046133348","display_name":"Benjamin Timmermans","orcid":"https://orcid.org/0000-0002-5130-4309"},"institutions":[{"id":"https://openalex.org/I4210148065","display_name":"IBM (Netherlands)","ror":"https://ror.org/055yb9v95","country_code":"NL","type":"company","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210148065"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Benjamin Timmermans","raw_affiliation_strings":["IBM Benelux CAS,Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"IBM Benelux CAS,Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210148065"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5106085231","display_name":"Chris Welty","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris Welty","raw_affiliation_strings":["Google Research,New York, New York, USA"],"affiliations":[{"raw_affiliation_string":"Google Research,New York, New York, USA","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5016381010"],"corresponding_institution_ids":["https://openalex.org/I1291425158"],"apc_list":null,"apc_paid":null,"fwci":0.6251,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.74931715,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1294","last_page":"1295"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9771999716758728,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9649856090545654},{"id":"https://openalex.org/keywords/diversity","display_name":"Diversity (politics)","score":0.7260260581970215},{"id":"https://openalex.org/keywords/ambiguity","display_name":"Ambiguity","score":0.702328085899353},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6682401895523071},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.6573601365089417},{"id":"https://openalex.org/keywords/social-media","display_name":"Social media","score":0.5287681818008423},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4980030059814453},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.4825660288333893},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.46906065940856934},{"id":"https://openalex.org/keywords/multitude","display_name":"Multitude","score":0.465143084526062},{"id":"https://openalex.org/keywords/citizen-science","display_name":"Citizen science","score":0.44862180948257446},{"id":"https://openalex.org/keywords/dissenting-opinion","display_name":"Dissenting opinion","score":0.423915833234787},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.22097468376159668},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.12280896306037903},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.11435958743095398}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9649856090545654},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.7260260581970215},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.702328085899353},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6682401895523071},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.6573601365089417},{"id":"https://openalex.org/C518677369","wikidata":"https://www.wikidata.org/wiki/Q202833","display_name":"Social media","level":2,"score":0.5287681818008423},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4980030059814453},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.4825660288333893},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.46906065940856934},{"id":"https://openalex.org/C2780565519","wikidata":"https://www.wikidata.org/wiki/Q1208937","display_name":"Multitude","level":2,"score":0.465143084526062},{"id":"https://openalex.org/C197352329","wikidata":"https://www.wikidata.org/wiki/Q1093434","display_name":"Citizen science","level":2,"score":0.44862180948257446},{"id":"https://openalex.org/C56617239","wikidata":"https://www.wikidata.org/wiki/Q1092720","display_name":"Dissenting opinion","level":2,"score":0.423915833234787},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.22097468376159668},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.12280896306037903},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.11435958743095398},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3308558.3320096","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308558.3320096","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3308558.3320096","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of The 2019 World Wide Web Conference","raw_type":"proceedings-article"},{"id":"pmh:oai:research.vu.nl:publications/8fc62aba-1459-40ea-b271-b20e55a88a2c","is_oa":true,"landing_page_url":"http://www.scopus.com/inward/record.url?scp=85066899250&partnerID=8YFLogxK","pdf_url":null,"source":{"id":"https://openalex.org/S4306401107","display_name":"VU Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I865915315","host_organization_name":"Vrije Universiteit Amsterdam","host_organization_lineage":["https://openalex.org/I865915315"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Aroyo, L, Szl\u00e1vik, Z, Dumitrache, A, Timmermans, B, Inel, O & Welty, C 2019, Crowdsourcing inclusivity : Dealing with diversity of opinions, perspectives and ambiguity in annotated data: The Crowdtruth tutorial. in The Web Conference 2019 : Proceedings of The World Wide Web Conference WWW 2019. Association for Computing Machinery, Inc, pp. 1294-1295, 2019 World Wide Web Conference, WWW 2019, San Francisco, United States, 13/05/19. https://doi.org/10.1145/3308558.3320096","raw_type":"contributionToPeriodical"},{"id":"pmh:vu:oai:research.vu.nl:publications/8fc62aba-1459-40ea-b271-b20e55a88a2c","is_oa":false,"landing_page_url":"https://research.vu.nl/en/publications/8fc62aba-1459-40ea-b271-b20e55a88a2c","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"The Web Conference 2019: Proceedings of The World Wide Web Conference WWW 2019, 1294 - 1295","raw_type":"info:eu-repo/semantics/conferencepaper"}],"best_oa_location":{"id":"doi:10.1145/3308558.3320096","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308558.3320096","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3308558.3320096","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of The 2019 World Wide Web Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/11","display_name":"Sustainable cities and communities","score":0.46000000834465027}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2945048417.pdf","grobid_xml":"https://content.openalex.org/works/W2945048417.grobid-xml"},"referenced_works_count":14,"referenced_works":["https://openalex.org/W1629525391","https://openalex.org/W2022881688","https://openalex.org/W2573154299","https://openalex.org/W2574781439","https://openalex.org/W2607311634","https://openalex.org/W2614403205","https://openalex.org/W2765320432","https://openalex.org/W2770047107","https://openalex.org/W2798706400","https://openalex.org/W2891077499","https://openalex.org/W2897694452","https://openalex.org/W2962732110","https://openalex.org/W3124912334","https://openalex.org/W4230282492"],"related_works":["https://openalex.org/W3207284869","https://openalex.org/W2055935989","https://openalex.org/W2883557297","https://openalex.org/W4230163270","https://openalex.org/W2621908761","https://openalex.org/W2417432647","https://openalex.org/W2254253365","https://openalex.org/W2371114124","https://openalex.org/W2168950483","https://openalex.org/W2555391182"],"abstract_inverted_index":{"In":[0],"this":[1,123,167],"tutorial,":[2],"we":[3,55],"introduce":[4,127],"a":[5,65,75,104,131],"novel":[6,132],"crowdsourcing":[7,78,135,161],"methodology":[8],"called":[9],"CrowdTruth":[10,17,73],"[1,":[11],"9].":[12],"The":[13,120],"central":[14],"characteristic":[15],"of":[16,30,71,106,122,139,142,171],"is":[18,74,125,147],"harnessing":[19],"the":[20,27,128,140,150,176,183],"diversity":[21,141,159],"in":[22,103,160],"human":[23],"interpretation":[24],"to":[25,126,130,134,149,175,181],"capture":[26],"wide":[28],"range":[29],"opinions":[31,143],"and":[32,34,40,47,67,84,98,102,117,144,158,188],"perspectives,":[33],"thus":[35],"provide":[36],"more":[37,68,168,185],"reliable,":[38,186],"realistic":[39],"inclusive":[41],"real-world":[42],"annotated":[43],"data":[44],"for":[45],"training":[46],"evaluating":[48],"machine":[49],"learning":[50],"components.":[51],"Unlike":[52],"other":[53],"methods,":[54],"do":[56],"not":[57],"discard":[58],"dissenting":[59],"votes,":[60],"but":[61],"incorporate":[62],"them":[63],"into":[64],"richer":[66],"continuous":[69],"representation":[70],"truth.":[72],"widely":[76],"used":[77],"methodology1":[79],"adopted":[80],"by":[81],"industrial":[82],"partners":[83],"public":[85],"organizations":[86],"such":[87,108],"as":[88,109,152],"Google,":[89],"IBM,":[90],"New":[91],"York":[92],"Times,":[93],"Cleveland":[94],"Clinic,":[95],"Crowdynews,":[96],"Sound":[97],"Vision":[99],"archive,":[100],"Rijksmuseum,":[101],"multitude":[105],"domains":[107],"AI,":[110],"news,":[111],"medicine,":[112],"social":[113,118],"media,":[114],"cultural":[115],"heritage,":[116],"sciences.":[119],"goal":[121],"tutorial":[124],"audience":[129],"approach":[133],"that":[136,146,154],"takes":[137],"advantage":[138],"perspectives":[145],"inherent":[148],"Web,":[151],"methods":[153],"deal":[155],"with":[156],"disagreement":[157],"have":[162],"become":[163],"increasingly":[164],"popular.":[165],"Creating":[166],"complex":[169],"notion":[170],"truth":[172],"contributes":[173],"directly":[174],"larger":[177],"discussion":[178],"on":[179],"how":[180],"make":[182],"Web":[184],"diverse":[187],"inclusive.":[189]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2026-04-03T22:45:19.894376","created_date":"2019-05-29T00:00:00"}