{"id":"https://openalex.org/W4308764909","doi":"https://doi.org/10.1145/3555634","title":"Towards Transparency in Dermatology Image Datasets with Skin Tone Annotations by Experts, Crowds, and an Algorithm","display_name":"Towards Transparency in Dermatology Image Datasets with Skin Tone Annotations by Experts, Crowds, and an Algorithm","publication_year":2022,"publication_date":"2022-11-07","ids":{"openalex":"https://openalex.org/W4308764909","doi":"https://doi.org/10.1145/3555634"},"language":"en","primary_location":{"id":"doi:10.1145/3555634","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3555634","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3555634","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3555634","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042355795","display_name":"Matthew Groh","orcid":"https://orcid.org/0000-0002-9029-0157"},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Matthew Groh","raw_affiliation_strings":["Massachusetts Institute of Technology, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology, Cambridge, MA, USA","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011698674","display_name":"Caleb Harris","orcid":null},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Caleb Harris","raw_affiliation_strings":["Massachusetts Institute of Technology, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology, Cambridge, MA, USA","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014300312","display_name":"Roxana Daneshjou","orcid":"https://orcid.org/0000-0001-7988-9356"},"institutions":[{"id":"https://openalex.org/I4210137306","display_name":"Stanford Medicine","ror":"https://ror.org/03mtd9a03","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210137306","https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Roxana Daneshjou","raw_affiliation_strings":["Stanford, Stanford, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford, Stanford, CA, USA","institution_ids":["https://openalex.org/I4210137306"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056256357","display_name":"Omar Badri","orcid":"https://orcid.org/0000-0003-2662-472X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Omar Badri","raw_affiliation_strings":["Northeast Dermatology Associates, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeast Dermatology Associates, Boston, MA, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5073580328","display_name":"Arash Koochek","orcid":null},"institutions":[{"id":"https://openalex.org/I2802412784","display_name":"Banner Health","ror":"https://ror.org/039wwwz66","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I2802412784"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Arash Koochek","raw_affiliation_strings":["Banner Health, Phoenix, AZ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Banner Health, Phoenix, AZ, USA","institution_ids":["https://openalex.org/I2802412784"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5042355795"],"corresponding_institution_ids":["https://openalex.org/I63966007"],"apc_list":null,"apc_paid":null,"fwci":4.5759,"has_fulltext":true,"cited_by_count":51,"citation_normalized_percentile":{"value":0.96301656,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"6","issue":"CSCW2","first_page":"1","last_page":"26"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10392","display_name":"Cutaneous Melanoma Detection and Management","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/2730","display_name":"Oncology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T10392","display_name":"Cutaneous Melanoma Detection and Management","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/2730","display_name":"Oncology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9409000277519226,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transparency","display_name":"Transparency (behavior)","score":0.753422200679779},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7333819270133972},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.6423895359039307},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5284010171890259},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.48143714666366577},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.46264398097991943},{"id":"https://openalex.org/keywords/certification","display_name":"Certification","score":0.4598686397075653},{"id":"https://openalex.org/keywords/medical-diagnosis","display_name":"Medical diagnosis","score":0.4296375811100006},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.40623682737350464},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.17656773328781128},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.1421559453010559},{"id":"https://openalex.org/keywords/pathology","display_name":"Pathology","score":0.1013399064540863}],"concepts":[{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.753422200679779},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7333819270133972},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.6423895359039307},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5284010171890259},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.48143714666366577},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.46264398097991943},{"id":"https://openalex.org/C46304622","wikidata":"https://www.wikidata.org/wiki/Q374814","display_name":"Certification","level":2,"score":0.4598686397075653},{"id":"https://openalex.org/C534262118","wikidata":"https://www.wikidata.org/wiki/Q177719","display_name":"Medical diagnosis","level":2,"score":0.4296375811100006},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40623682737350464},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.17656773328781128},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.1421559453010559},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.1013399064540863},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3555634","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3555634","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3555634","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3555634","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3555634","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3555634","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4308764909.pdf","grobid_xml":"https://content.openalex.org/works/W4308764909.grobid-xml"},"referenced_works_count":84,"referenced_works":["https://openalex.org/W404240460","https://openalex.org/W582173391","https://openalex.org/W1523116602","https://openalex.org/W1819662813","https://openalex.org/W1962580118","https://openalex.org/W1981474567","https://openalex.org/W2007018772","https://openalex.org/W2007602682","https://openalex.org/W2009681737","https://openalex.org/W2057780053","https://openalex.org/W2147603330","https://openalex.org/W2256829900","https://openalex.org/W2574220820","https://openalex.org/W2581082771","https://openalex.org/W2593035626","https://openalex.org/W2604451061","https://openalex.org/W2752437500","https://openalex.org/W2777647957","https://openalex.org/W2791454671","https://openalex.org/W2791924245","https://openalex.org/W2800068874","https://openalex.org/W2897042519","https://openalex.org/W2905060314","https://openalex.org/W2918341242","https://openalex.org/W2941766203","https://openalex.org/W2942399136","https://openalex.org/W2970837303","https://openalex.org/W2971867419","https://openalex.org/W2979893369","https://openalex.org/W2980349812","https://openalex.org/W2981869278","https://openalex.org/W2984904339","https://openalex.org/W2998175747","https://openalex.org/W3003162384","https://openalex.org/W3003599720","https://openalex.org/W3004483087","https://openalex.org/W3007910004","https://openalex.org/W3012442931","https://openalex.org/W3014403957","https://openalex.org/W3019100240","https://openalex.org/W3024132275","https://openalex.org/W3025011581","https://openalex.org/W3028830016","https://openalex.org/W3029264758","https://openalex.org/W3036298167","https://openalex.org/W3036877591","https://openalex.org/W3036969299","https://openalex.org/W3046215811","https://openalex.org/W3093541323","https://openalex.org/W3094217062","https://openalex.org/W3094294413","https://openalex.org/W3100279624","https://openalex.org/W3104831984","https://openalex.org/W3106328160","https://openalex.org/W3112374793","https://openalex.org/W3118813946","https://openalex.org/W3119394424","https://openalex.org/W3119689140","https://openalex.org/W3125798375","https://openalex.org/W3133631714","https://openalex.org/W3143784018","https://openalex.org/W3159814022","https://openalex.org/W3160122248","https://openalex.org/W3162428030","https://openalex.org/W3162447981","https://openalex.org/W3163469193","https://openalex.org/W3170772782","https://openalex.org/W3175183715","https://openalex.org/W3189849087","https://openalex.org/W3199500316","https://openalex.org/W3199781855","https://openalex.org/W3204393347","https://openalex.org/W3206420877","https://openalex.org/W3207598588","https://openalex.org/W3207830467","https://openalex.org/W3209972423","https://openalex.org/W3210506295","https://openalex.org/W3212368439","https://openalex.org/W3215200385","https://openalex.org/W4205570006","https://openalex.org/W4212904094","https://openalex.org/W4247236552","https://openalex.org/W4283832306","https://openalex.org/W4290852327"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W1503094549","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W3207526114","https://openalex.org/W4286908577"],"abstract_inverted_index":{"While":[0],"artificial":[1],"intelligence":[2],"(AI)":[3],"holds":[4],"promise":[5],"for":[6,80,162,230,272],"supporting":[7],"healthcare":[8],"providers":[9],"and":[10,34,40,94,128,156,185,269],"improving":[11],"the":[12,22,30,63,66,101,143,168,178,182,188,196,207,211,251,265],"accuracy":[13],"of":[14,19,24,32,44,65,74,87,103,119,122,187,253,267,276],"medical":[15],"diagnoses,":[16],"a":[17,54,72,78,116],"lack":[18],"transparency":[20,244],"in":[21,84],"composition":[23],"datasets":[25,43],"exposes":[26],"AI":[27,59],"models":[28],"to":[29,77,99,152,177,245],"possibility":[31],"unintentional":[33],"avoidable":[35],"mistakes.":[36],"In":[37,97,191],"particular,":[38],"public":[39],"private":[41],"image":[42,233,278],"dermatological":[45],"conditions":[46,124],"rarely":[47],"include":[48,134],"information":[49],"on":[50,107,115,225],"skin":[51,68,82,123,242],"color.":[52],"As":[53],"start":[55],"towards":[56],"increasing":[57],"transparency,":[58],"researchers":[60],"have":[61],"appropriated":[62],"use":[64],"Fitzpatrick":[67],"type":[69,243],"(FST)":[70],"from":[71,125],"measure":[73,79],"patient":[75],"photosensitivity":[76],"estimating":[81],"tone":[83],"algorithmic":[85,140],"audits":[86],"computer":[88],"vision":[89],"applications":[90],"including":[91,260],"facial":[92],"recognition":[93],"dermatology":[95,130,246],"diagnosis.":[96],"order":[98],"understand":[100],"variability":[102],"estimated":[104,153,164],"FST":[105,112,154],"annotations":[106,200,209,213,275],"images,":[108],"we":[109,193,249],"compare":[110],"several":[111],"annotation":[113,136,141],"methods":[114,133],"diverse":[117],"set":[118],"460":[120],"images":[121],"both":[126],"textbooks":[127],"online":[129],"atlases.":[131],"These":[132,219],"expert":[135,261],"by":[137],"board-certified":[138,173,183],"dermatologists,":[139],"via":[142],"Individual":[144],"Typology":[145],"Angle":[146],"algorithm,":[147],"which":[148],"is":[149,175],"then":[150],"converted":[151],"(ITA-FST),":[155],"two":[157],"crowd-sourced,":[158],"dynamic":[159,254],"consensus":[160,255],"protocols":[161,238,256],"annotating":[163,231],"FSTs.":[165],"We":[166],"find":[167,194],"inter-rater":[169,179],"reliability":[170,180],"between":[171,181],"three":[172],"dermatologists":[174,184],"comparable":[176],"either":[186],"crowdsourcing":[189],"methods.":[190],"contrast,":[192],"that":[195,201,222,263],"ITA-FST":[197,226],"method":[198],"produces":[199],"are":[202,214,227],"significantly":[203],"less":[204],"correlated":[205,215],"with":[206,216,257],"experts'":[208,212],"than":[210],"each":[217],"other.":[218],"results":[220],"demonstrate":[221],"algorithms":[223],"based":[224],"not":[228],"reliable":[229],"large-scale":[232],"datasets,":[234],"but":[235],"human-centered,":[236],"crowd-based":[237],"can":[239],"reliably":[240],"add":[241],"datasets.":[247,279],"Furthermore,":[248],"introduce":[250],"concept":[252],"tunable":[258],"parameters":[259],"review":[262],"increase":[264],"visibility":[266],"crowdwork":[268],"provide":[270],"guidance":[271],"future":[273],"crowdsourced":[274],"large":[277]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":16},{"year":2024,"cited_by_count":19},{"year":2023,"cited_by_count":9}],"updated_date":"2026-05-21T09:19:25.381259","created_date":"2025-10-10T00:00:00"}