{"id":"https://openalex.org/W4376988619","doi":"https://doi.org/10.1145/3593013.3594114","title":"Skin Deep: Investigating Subjectivity in Skin Tone Annotations for Computer Vision Benchmark Datasets","display_name":"Skin Deep: Investigating Subjectivity in Skin Tone Annotations for Computer Vision Benchmark Datasets","publication_year":2023,"publication_date":"2023-06-12","ids":{"openalex":"https://openalex.org/W4376988619","doi":"https://doi.org/10.1145/3593013.3594114"},"language":"en","primary_location":{"id":"doi:10.1145/3593013.3594114","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3593013.3594114","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 ACM Conference on Fairness Accountability and Transparency","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2305.09072","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051179990","display_name":"Teanna Barrett","orcid":"https://orcid.org/0000-0002-3259-112X"},"institutions":[{"id":"https://openalex.org/I137853757","display_name":"Howard University","ror":"https://ror.org/05gt1vc06","country_code":"US","type":"education","lineage":["https://openalex.org/I137853757"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Teanna Barrett","raw_affiliation_strings":["Department of Electrical Engineering & Computer Science, Howard University, USA"],"raw_orcid":"https://orcid.org/0000-0002-3259-112X","affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering & Computer Science, Howard University, USA","institution_ids":["https://openalex.org/I137853757"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037054163","display_name":"Quan Ze Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Quanze Chen","raw_affiliation_strings":["Allen School of Computer Science & Engineering, University of Washington, USA"],"raw_orcid":"https://orcid.org/0000-0002-6500-8922","affiliations":[{"raw_affiliation_string":"Allen School of Computer Science & Engineering, University of Washington, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037569091","display_name":"Amy X. Zhang","orcid":"https://orcid.org/0000-0001-9462-9835"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amy Zhang","raw_affiliation_strings":["Allen School of Computer Science & Engineering, University of Washington, USA"],"raw_orcid":"https://orcid.org/0000-0001-9462-9835","affiliations":[{"raw_affiliation_string":"Allen School of Computer Science & Engineering, University of Washington, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5051179990"],"corresponding_institution_ids":["https://openalex.org/I137853757"],"apc_list":null,"apc_paid":null,"fwci":1.7662,"has_fulltext":true,"cited_by_count":15,"citation_normalized_percentile":{"value":0.86898019,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1757","last_page":"1771"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10331","display_name":"Video Surveillance and Tracking Methods","score":0.9763000011444092,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10331","display_name":"Video Surveillance and Tracking Methods","score":0.9763000011444092,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11019","display_name":"Image Enhancement Techniques","score":0.972599983215332,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9506000280380249,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.7694281339645386},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7016298174858093},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6621955633163452},{"id":"https://openalex.org/keywords/tone","display_name":"Tone (literature)","score":0.63310307264328},{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.5525298118591309},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.434714138507843},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38031989336013794},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.32417505979537964},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.171548992395401},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.07942193746566772}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7694281339645386},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7016298174858093},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6621955633163452},{"id":"https://openalex.org/C2780583480","wikidata":"https://www.wikidata.org/wiki/Q1366327","display_name":"Tone (literature)","level":2,"score":0.63310307264328},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.5525298118591309},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.434714138507843},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38031989336013794},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.32417505979537964},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.171548992395401},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.07942193746566772},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3593013.3594114","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3593013.3594114","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 ACM Conference on Fairness Accountability and Transparency","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2305.09072","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.09072","pdf_url":"https://arxiv.org/pdf/2305.09072","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2305.09072","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.09072","pdf_url":"https://arxiv.org/pdf/2305.09072","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.4099999964237213,"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10"}],"awards":[{"id":"https://openalex.org/G8561224762","display_name":"Collaborative Research: SaTC: CORE: Large: Privacy-Preserving Abuse Prevention for Encrypted Communications Platforms","funder_award_id":"2120497","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4376988619.pdf","grobid_xml":"https://content.openalex.org/works/W4376988619.grobid-xml"},"referenced_works_count":98,"referenced_works":["https://openalex.org/W1650540693","https://openalex.org/W1834627138","https://openalex.org/W1847618513","https://openalex.org/W1978642336","https://openalex.org/W2007602682","https://openalex.org/W2094388549","https://openalex.org/W2114269021","https://openalex.org/W2135504089","https://openalex.org/W2469104253","https://openalex.org/W2520509592","https://openalex.org/W2523443110","https://openalex.org/W2595961471","https://openalex.org/W2736633948","https://openalex.org/W2748916233","https://openalex.org/W2759803525","https://openalex.org/W2764072425","https://openalex.org/W2768938399","https://openalex.org/W2783946743","https://openalex.org/W2788481061","https://openalex.org/W2792504228","https://openalex.org/W2803616268","https://openalex.org/W2807227384","https://openalex.org/W2810511387","https://openalex.org/W2871667416","https://openalex.org/W2890392482","https://openalex.org/W2912376035","https://openalex.org/W2915268314","https://openalex.org/W2922269948","https://openalex.org/W2954314661","https://openalex.org/W2958396160","https://openalex.org/W2969447056","https://openalex.org/W2982563701","https://openalex.org/W3006568518","https://openalex.org/W3012442931","https://openalex.org/W3013451997","https://openalex.org/W3028830016","https://openalex.org/W3029264758","https://openalex.org/W3043315258","https://openalex.org/W3093438907","https://openalex.org/W3093473767","https://openalex.org/W3094231124","https://openalex.org/W3111505403","https://openalex.org/W3111746712","https://openalex.org/W3112685743","https://openalex.org/W3127678221","https://openalex.org/W3130139798","https://openalex.org/W3132771424","https://openalex.org/W3134936382","https://openalex.org/W3135474405","https://openalex.org/W3137020077","https://openalex.org/W3137175206","https://openalex.org/W3140832216","https://openalex.org/W3154010931","https://openalex.org/W3155403969","https://openalex.org/W3163078977","https://openalex.org/W3164751134","https://openalex.org/W3170523166","https://openalex.org/W3175183715","https://openalex.org/W3175667647","https://openalex.org/W3186337419","https://openalex.org/W3187437888","https://openalex.org/W3190434573","https://openalex.org/W3207413983","https://openalex.org/W3207594020","https://openalex.org/W3207879356","https://openalex.org/W3208553288","https://openalex.org/W3208589729","https://openalex.org/W4200319912","https://openalex.org/W4200388791","https://openalex.org/W4200595449","https://openalex.org/W4200632307","https://openalex.org/W4205285111","https://openalex.org/W4206749241","https://openalex.org/W4210596498","https://openalex.org/W4212904094","https://openalex.org/W4213427026","https://openalex.org/W4214810323","https://openalex.org/W4221141821","https://openalex.org/W4221146266","https://openalex.org/W4226312557","https://openalex.org/W4226459344","https://openalex.org/W4250543126","https://openalex.org/W4250928742","https://openalex.org/W4280573535","https://openalex.org/W4283164137","https://openalex.org/W4283168161","https://openalex.org/W4283168787","https://openalex.org/W4285583839","https://openalex.org/W4286893053","https://openalex.org/W4287239856","https://openalex.org/W4288083800","https://openalex.org/W4288346602","https://openalex.org/W4290852327","https://openalex.org/W4297798776","https://openalex.org/W4308764909","https://openalex.org/W4312694272","https://openalex.org/W4313163853","https://openalex.org/W4394637340"],"related_works":["https://openalex.org/W2392768766","https://openalex.org/W2058118494","https://openalex.org/W2361861616","https://openalex.org/W2382021449","https://openalex.org/W2095118173","https://openalex.org/W2263699433","https://openalex.org/W2104269053","https://openalex.org/W2106424170","https://openalex.org/W2377979023","https://openalex.org/W1985426483"],"abstract_inverted_index":{"To":[0],"investigate":[1,107],"the":[2,34,75,78,82,89,98,104,108,148,157,182,187,191,201,210,227],"well-observed":[3],"racial":[4],"disparities":[5],"in":[6,81,97,138,164,171,181,189,200,226],"computer":[7,172],"vision":[8,173],"systems":[9],"that":[10,178],"analyze":[11,161],"images":[12],"of":[13,37,51,66,77,94,133,205,217,232],"humans,":[14],"researchers":[15,47],"have":[16,58],"turned":[17],"to":[18,74,106,128,160],"skin":[19,38,55,95,109,120,134,139,152,168,192,218,237],"tone":[20,39,56,96,110,121,135,140,153,169,193,219],"as":[21,113,186],"a":[22,49,114,206],"more":[23],"objective":[24],"annotation":[25,40,60,83,111,122,126,141,170,183],"than":[26],"race":[27,155],"metadata":[28],"for":[29,167,223,234],"fairness":[30],"performance":[31],"evaluations.":[32],"However,":[33],"current":[35,165],"state":[36],"procedures":[41,123,166,233],"is":[42,72,103,195],"highly":[43],"varied.":[44],"For":[45],"instance,":[46],"use":[48],"range":[50],"untested":[52],"scales":[53],"and":[54,62,86,91,124,154,156,214,230],"categories,":[57],"unclear":[59],"procedures,":[61],"provide":[63],"inadequate":[64],"analyses":[65],"uncertainty.":[67],"In":[68],"addition,":[69],"little":[70],"attention":[71],"paid":[73],"positionality":[76],"humans":[79],"involved":[80],"process\u2014both":[84],"designers":[85],"annotators":[87],"alike\u2014and":[88],"historical":[90],"sociological":[92],"context":[93,199],"United":[99],"States.":[100],"Our":[101,143,175],"work":[102],"first":[105],"process":[112],"sociotechnical":[115],"project.":[116],"We":[117,221],"surveyed":[118],"recent":[119],"conducted":[125],"experiments":[127,176],"examine":[129],"how":[130],"subjective":[131],"understandings":[132],"are":[136],"embedded":[137],"procedures.":[142],"systematic":[144],"literature":[145],"review":[146],"revealed":[147],"uninterrogated":[149],"association":[150],"between":[151],"limited":[158],"effort":[159],"annotator":[162],"uncertainty":[163,216],"evaluation.":[174],"demonstrated":[177],"design":[179],"decisions":[180],"procedure":[184],"such":[185],"order":[188],"which":[190],"scale":[194],"presented":[196],"or":[197],"additional":[198],"image":[202],"(i.e.,":[203],"presence":[204],"face)":[207],"significantly":[208],"affected":[209],"resulting":[211],"inter-annotator":[212],"agreement":[213],"individual":[215],"annotations.":[220],"call":[222],"greater":[224],"reflexivity":[225],"design,":[228],"analysis,":[229],"documentation":[231],"evaluation":[235],"using":[236],"tone.":[238]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":6}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}