{"id":"https://openalex.org/W4366588458","doi":"https://doi.org/10.1145/3544548.3580645","title":"A hunt for the Snark: Annotator Diversity in Data Practices","display_name":"A hunt for the Snark: Annotator Diversity in Data Practices","publication_year":2023,"publication_date":"2023-04-19","ids":{"openalex":"https://openalex.org/W4366588458","doi":"https://doi.org/10.1145/3544548.3580645"},"language":"en","primary_location":{"id":"doi:10.1145/3544548.3580645","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3580645","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3580645","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3580645","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007266018","display_name":"Shivani Kapania","orcid":"https://orcid.org/0000-0002-0152-4311"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Shivani Kapania","raw_affiliation_strings":["Google Research, India"],"affiliations":[{"raw_affiliation_string":"Google Research, India","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029562752","display_name":"Alex Taylor","orcid":"https://orcid.org/0000-0001-6311-3967"},"institutions":[{"id":"https://openalex.org/I180825142","display_name":"City, University of London","ror":"https://ror.org/04489at23","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I180825142"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Alex S Taylor","raw_affiliation_strings":["Centre for HCI Design, City, University of London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Centre for HCI Design, City, University of London, United Kingdom","institution_ids":["https://openalex.org/I180825142"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052316777","display_name":"Ding Wang","orcid":"https://orcid.org/0000-0002-5037-5017"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding Wang","raw_affiliation_strings":["Google, Singapore"],"affiliations":[{"raw_affiliation_string":"Google, Singapore","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5007266018"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":11.1912,"has_fulltext":true,"cited_by_count":44,"citation_normalized_percentile":{"value":0.98714801,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"15"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9606000185012817,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/diversity","display_name":"Diversity (politics)","score":0.7486659288406372},{"id":"https://openalex.org/keywords/objectivity","display_name":"Objectivity (philosophy)","score":0.5748134255409241},{"id":"https://openalex.org/keywords/subjectivity","display_name":"Subjectivity","score":0.5495017766952515},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5370859503746033},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4933268129825592},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.34377577900886536},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3222891092300415},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.2631832957267761},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.19849944114685059}],"concepts":[{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.7486659288406372},{"id":"https://openalex.org/C2482559","wikidata":"https://www.wikidata.org/wiki/Q206330","display_name":"Objectivity (philosophy)","level":2,"score":0.5748134255409241},{"id":"https://openalex.org/C202889954","wikidata":"https://www.wikidata.org/wiki/Q1139554","display_name":"Subjectivity","level":2,"score":0.5495017766952515},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5370859503746033},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4933268129825592},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.34377577900886536},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3222891092300415},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.2631832957267761},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.19849944114685059},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3544548.3580645","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3580645","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3580645","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.ed.ac.uk:openaire/cb816be3-4122-4162-9a29-fa14f349607b","is_oa":true,"landing_page_url":"https://www.research.ed.ac.uk/en/publications/cb816be3-4122-4162-9a29-fa14f349607b","pdf_url":"https://www.research.ed.ac.uk/files/410171055/A_hunt_for_KAPANIA_VOR_CC_BY.pdf","source":{"id":"https://openalex.org/S4406922455","display_name":"Edinburgh Research Explorer","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Kapania, S, Taylor, A S & Wang, D 2023, A hunt for the Snark : Annotator Diversity in Data Practices. in CHI '23 : Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems., 133, Conference on Human Factors in Computing Systems - Proceedings, pp. 1-15, 2023 CHI Conference on Human Factors in Computing Systems, Hamburg, Hamburg, Germany, 23/04/23. https://doi.org/10.1145/3544548.3580645","raw_type":"contributionToPeriodical"},{"id":"pmh:oai:pure.ed.ac.uk:publications/cb816be3-4122-4162-9a29-fa14f349607b","is_oa":true,"landing_page_url":"https://hdl.handle.net/20.500.11820/cb816be3-4122-4162-9a29-fa14f349607b","pdf_url":null,"source":{"id":"https://openalex.org/S4406922455","display_name":"Edinburgh Research Explorer","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":""}],"best_oa_location":{"id":"doi:10.1145/3544548.3580645","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3580645","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3580645","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4366588458.pdf","grobid_xml":"https://content.openalex.org/works/W4366588458.grobid-xml"},"referenced_works_count":74,"referenced_works":["https://openalex.org/W110542310","https://openalex.org/W118259468","https://openalex.org/W123719442","https://openalex.org/W200342760","https://openalex.org/W640074922","https://openalex.org/W1557323065","https://openalex.org/W1813736802","https://openalex.org/W1962580118","https://openalex.org/W1986792585","https://openalex.org/W1991612274","https://openalex.org/W1996149054","https://openalex.org/W1996934409","https://openalex.org/W2007018772","https://openalex.org/W2027400672","https://openalex.org/W2102452846","https://openalex.org/W2106743565","https://openalex.org/W2120096569","https://openalex.org/W2125943921","https://openalex.org/W2136222780","https://openalex.org/W2164124780","https://openalex.org/W2285144430","https://openalex.org/W2290009368","https://openalex.org/W2329395632","https://openalex.org/W2588064451","https://openalex.org/W2599032673","https://openalex.org/W2783218188","https://openalex.org/W2795743913","https://openalex.org/W2795975316","https://openalex.org/W2893425640","https://openalex.org/W2898898463","https://openalex.org/W2899134503","https://openalex.org/W2910548926","https://openalex.org/W2911227954","https://openalex.org/W2920114910","https://openalex.org/W2941766203","https://openalex.org/W2944944282","https://openalex.org/W2953522645","https://openalex.org/W2957654274","https://openalex.org/W2979746687","https://openalex.org/W2988571156","https://openalex.org/W2997588435","https://openalex.org/W3009312460","https://openalex.org/W3030406212","https://openalex.org/W3093541323","https://openalex.org/W3101243562","https://openalex.org/W3118813946","https://openalex.org/W3133631714","https://openalex.org/W3158349484","https://openalex.org/W3162866579","https://openalex.org/W3163078977","https://openalex.org/W3163469193","https://openalex.org/W3185692625","https://openalex.org/W3196853733","https://openalex.org/W3206428286","https://openalex.org/W3212368439","https://openalex.org/W3213556453","https://openalex.org/W4205570006","https://openalex.org/W4210702550","https://openalex.org/W4224981752","https://openalex.org/W4225001019","https://openalex.org/W4225006216","https://openalex.org/W4225086340","https://openalex.org/W4243761327","https://openalex.org/W4250532311","https://openalex.org/W4280609065","https://openalex.org/W4283155630","https://openalex.org/W4283166427","https://openalex.org/W4288079542","https://openalex.org/W4288083705","https://openalex.org/W4288083800","https://openalex.org/W4292341621","https://openalex.org/W4302301976","https://openalex.org/W4309618884","https://openalex.org/W4309619003"],"related_works":["https://openalex.org/W2391158797","https://openalex.org/W1920616658","https://openalex.org/W154900787","https://openalex.org/W2347990546","https://openalex.org/W2390324744","https://openalex.org/W3193146728","https://openalex.org/W2583808709","https://openalex.org/W2393314469","https://openalex.org/W2971756891","https://openalex.org/W3161154222"],"abstract_inverted_index":{"Diversity":[0],"in":[1,24,62,80,89,111,143],"datasets":[2],"is":[3],"a":[4,39,108,128],"key":[5],"component":[6],"to":[7,31,58,85,103,124],"building":[8],"responsible":[9],"AI/ML.":[10],"Despite":[11],"this":[12,122],"recognition,":[13],"we":[14,132],"know":[15],"little":[16],"about":[17],"the":[18,21,29,63,76,81,86,100],"diversity":[19,33,61,106,142],"among":[20],"annotators":[22],"involved":[23],"data":[25,112,144],"production.":[26],"We":[27,93],"investigated":[28],"approaches":[30],"annotator":[32,51,82,105,139],"through":[34,72],"16":[35],"semi-structured":[36],"interviews":[37],"and":[38,99,116,141],"survey":[40],"with":[41],"44":[42],"AI/ML":[43],"practitioners.":[44],"While":[45],"practitioners":[46],"described":[47],"nuanced":[48],"understandings":[49],"of":[50,68,78,127,130,136],"diversity,":[52],"they":[53],"rarely":[54],"designed":[55],"dataset":[56],"production":[57],"account":[59],"for":[60,138],"annotation":[64],"process.":[65],"The":[66],"lack":[67,77],"action":[69],"was":[70],"explained":[71],"operational":[73,97],"barriers:":[74],"from":[75],"visibility":[79],"hiring":[83],"process,":[84],"conceptual":[87],"difficulty":[88],"incorporating":[90],"worker":[91],"diversity.":[92],"argue":[94],"that":[95],"such":[96],"barriers":[98],"widespread":[101],"resistance":[102],"accommodating":[104],"surface":[107],"prevailing":[109],"logic":[110,123],"practices\u2014where":[113],"neutrality,":[114],"objectivity":[115],"\u2018representationalist":[117],"thinking\u2019":[118],"dominate.":[119],"By":[120],"understanding":[121],"be":[125],"part":[126],"regime":[129],"existence,":[131],"explore":[133],"alternative":[134],"ways":[135],"accounting":[137],"subjectivity":[140],"practices.":[145]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":20},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":4}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}