{"id":"https://openalex.org/W4386350446","doi":"https://doi.org/10.1109/isbi53787.2023.10230646","title":"Nonexpert Crowds Outperform Expert Individuals in Diagnostic Accuracy on a Skin Lesion Diagnosis Task","display_name":"Nonexpert Crowds Outperform Expert Individuals in Diagnostic Accuracy on a Skin Lesion Diagnosis Task","publication_year":2023,"publication_date":"2023-04-18","ids":{"openalex":"https://openalex.org/W4386350446","doi":"https://doi.org/10.1109/isbi53787.2023.10230646"},"language":"en","primary_location":{"id":"doi:10.1109/isbi53787.2023.10230646","is_oa":false,"landing_page_url":"https://doi.org/10.1109/isbi53787.2023.10230646","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE 20th International Symposium on Biomedical Imaging (ISBI)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034068566","display_name":"Erik Duhaime","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Erik P Duhaime","raw_affiliation_strings":["Centaur Labs,Boston,MA,02118"],"affiliations":[{"raw_affiliation_string":"Centaur Labs,Boston,MA,02118","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061852666","display_name":"Mike Jin","orcid":"https://orcid.org/0000-0001-7237-6697"},"institutions":[{"id":"https://openalex.org/I1283280774","display_name":"Brigham and Women's Hospital","ror":"https://ror.org/04b6nzv94","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283280774","https://openalex.org/I48633490"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mike Jin","raw_affiliation_strings":["Centaur Labs,Boston,MA,02118","Brigham and Women's Hospital, Boston, MA"],"affiliations":[{"raw_affiliation_string":"Centaur Labs,Boston,MA,02118","institution_ids":[]},{"raw_affiliation_string":"Brigham and Women's Hospital, Boston, MA","institution_ids":["https://openalex.org/I1283280774"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076126200","display_name":"Taylor Moulton","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Taylor Moulton","raw_affiliation_strings":["Centaur Labs,Boston,MA,02118"],"affiliations":[{"raw_affiliation_string":"Centaur Labs,Boston,MA,02118","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087476551","display_name":"Jochen Weber","orcid":"https://orcid.org/0000-0002-2397-4954"},"institutions":[{"id":"https://openalex.org/I1334819555","display_name":"Memorial Sloan Kettering Cancer Center","ror":"https://ror.org/02yrq0923","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1334819555"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jochen Weber","raw_affiliation_strings":["Memorial Sloan Kettering Cancer Center,Dermatology Service,New York,NY,10065"],"affiliations":[{"raw_affiliation_string":"Memorial Sloan Kettering Cancer Center,Dermatology Service,New York,NY,10065","institution_ids":["https://openalex.org/I1334819555"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014554785","display_name":"Nicholas R. Kurtansky","orcid":"https://orcid.org/0000-0002-6745-0386"},"institutions":[{"id":"https://openalex.org/I1334819555","display_name":"Memorial Sloan Kettering Cancer Center","ror":"https://ror.org/02yrq0923","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1334819555"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nicholas R Kurtansky","raw_affiliation_strings":["Memorial Sloan Kettering Cancer Center,Dermatology Service,New York,NY,10065"],"affiliations":[{"raw_affiliation_string":"Memorial Sloan Kettering Cancer Center,Dermatology Service,New York,NY,10065","institution_ids":["https://openalex.org/I1334819555"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004712523","display_name":"Allan C. Halpern","orcid":"https://orcid.org/0000-0001-7320-1901"},"institutions":[{"id":"https://openalex.org/I1334819555","display_name":"Memorial Sloan Kettering Cancer Center","ror":"https://ror.org/02yrq0923","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1334819555"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Allan Halpern","raw_affiliation_strings":["Memorial Sloan Kettering Cancer Center,Dermatology Service,New York,NY,10065"],"affiliations":[{"raw_affiliation_string":"Memorial Sloan Kettering Cancer Center,Dermatology Service,New York,NY,10065","institution_ids":["https://openalex.org/I1334819555"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5083011959","display_name":"Veronica Rotemberg","orcid":"https://orcid.org/0000-0003-0639-2677"},"institutions":[{"id":"https://openalex.org/I1334819555","display_name":"Memorial Sloan Kettering Cancer Center","ror":"https://ror.org/02yrq0923","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1334819555"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Veronica Rotemberg","raw_affiliation_strings":["Memorial Sloan Kettering Cancer Center,Dermatology Service,New York,NY,10065"],"affiliations":[{"raw_affiliation_string":"Memorial Sloan Kettering Cancer Center,Dermatology Service,New York,NY,10065","institution_ids":["https://openalex.org/I1334819555"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5034068566"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3749,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.84139129,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10392","display_name":"Cutaneous Melanoma Detection and Management","score":0.9889000058174133,"subfield":{"id":"https://openalex.org/subfields/2730","display_name":"Oncology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T10392","display_name":"Cutaneous Melanoma Detection and Management","score":0.9889000058174133,"subfield":{"id":"https://openalex.org/subfields/2730","display_name":"Oncology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10862","display_name":"AI in cancer detection","score":0.9592999815940857,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11289","display_name":"Single-cell and spatial transcriptomics","score":0.9495000243186951,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9151808619499207},{"id":"https://openalex.org/keywords/receiver-operating-characteristic","display_name":"Receiver operating characteristic","score":0.6854475736618042},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6514475345611572},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.615151047706604},{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.596482515335083},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5324434041976929},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.48667818307876587},{"id":"https://openalex.org/keywords/skin-lesion","display_name":"Skin lesion","score":0.48291894793510437},{"id":"https://openalex.org/keywords/diagnostic-accuracy","display_name":"Diagnostic accuracy","score":0.4663170278072357},{"id":"https://openalex.org/keywords/majority-rule","display_name":"Majority rule","score":0.4481654167175293},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.42284369468688965},{"id":"https://openalex.org/keywords/area-under-curve","display_name":"Area under curve","score":0.4124511182308197},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.32974931597709656},{"id":"https://openalex.org/keywords/medical-physics","display_name":"Medical physics","score":0.3251012861728668},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.2569907605648041},{"id":"https://openalex.org/keywords/radiology","display_name":"Radiology","score":0.1877642273902893},{"id":"https://openalex.org/keywords/dermatology","display_name":"Dermatology","score":0.17386621236801147},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08131551742553711}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9151808619499207},{"id":"https://openalex.org/C58471807","wikidata":"https://www.wikidata.org/wiki/Q327120","display_name":"Receiver operating characteristic","level":2,"score":0.6854475736618042},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6514475345611572},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.615151047706604},{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.596482515335083},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5324434041976929},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48667818307876587},{"id":"https://openalex.org/C2988168687","wikidata":"https://www.wikidata.org/wiki/Q949302","display_name":"Skin lesion","level":2,"score":0.48291894793510437},{"id":"https://openalex.org/C3020132585","wikidata":"https://www.wikidata.org/wiki/Q2671652","display_name":"Diagnostic accuracy","level":2,"score":0.4663170278072357},{"id":"https://openalex.org/C153668964","wikidata":"https://www.wikidata.org/wiki/Q27636","display_name":"Majority rule","level":2,"score":0.4481654167175293},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.42284369468688965},{"id":"https://openalex.org/C3020225094","wikidata":"https://www.wikidata.org/wiki/Q80091","display_name":"Area under curve","level":3,"score":0.4124511182308197},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.32974931597709656},{"id":"https://openalex.org/C19527891","wikidata":"https://www.wikidata.org/wiki/Q1120908","display_name":"Medical physics","level":1,"score":0.3251012861728668},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.2569907605648041},{"id":"https://openalex.org/C126838900","wikidata":"https://www.wikidata.org/wiki/Q77604","display_name":"Radiology","level":1,"score":0.1877642273902893},{"id":"https://openalex.org/C16005928","wikidata":"https://www.wikidata.org/wiki/Q171171","display_name":"Dermatology","level":1,"score":0.17386621236801147},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08131551742553711},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C112705442","wikidata":"https://www.wikidata.org/wiki/Q323936","display_name":"Pharmacokinetics","level":2,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/isbi53787.2023.10230646","is_oa":false,"landing_page_url":"https://doi.org/10.1109/isbi53787.2023.10230646","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE 20th International Symposium on Biomedical Imaging (ISBI)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals","score":0.5199999809265137}],"awards":[],"funders":[{"id":"https://openalex.org/F4320337472","display_name":"National Center for Advancing Translational Sciences","ror":"https://ror.org/04pw6fb54"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W1912982817","https://openalex.org/W2158392329","https://openalex.org/W2581082771","https://openalex.org/W2952971376","https://openalex.org/W3007935259","https://openalex.org/W3014403957","https://openalex.org/W3131880546","https://openalex.org/W3132400453","https://openalex.org/W4300485340","https://openalex.org/W6746693533"],"related_works":["https://openalex.org/W4240200267","https://openalex.org/W1511510665","https://openalex.org/W1524661185","https://openalex.org/W2078823605","https://openalex.org/W2500095415","https://openalex.org/W2097922264","https://openalex.org/W2282342021","https://openalex.org/W4233026749","https://openalex.org/W2391774853","https://openalex.org/W4386934839"],"abstract_inverted_index":{"A":[0],"recent":[1],"study":[2],"[1]":[3],"showed":[4],"that":[5,123],"individual":[6],"physicians":[7],"with":[8],"at":[9],"least":[10],"ten":[11],"years":[12],"of":[13,75,89,100,119],"experience":[14],"as":[15],"dermatologists":[16],"achieved":[17,70],"74.7%":[18],"accuracy":[19,74],"on":[20,50],"average":[21],"in":[22,116],"labeling":[23],"images":[24],"from":[25],"the":[26,51,58,84,117],"multiclass":[27,79],"International":[28],"Skin":[29],"Imaging":[30],"Collaboration":[31],"(ISIC)":[32],"2018":[33],"challenge":[34],"dataset.":[35],"Using":[36],"a":[37,67,71,78],"novel":[38],"gamified":[39,112],"crowdsourcing":[40],"method,":[41],"we":[42],"collected":[43],"144,383":[44],"nonexpert":[45],"opinions":[46],"over":[47],"two":[48],"weeks":[49],"medical":[52,126],"image":[53],"annotation":[54],"platform":[55],"DiagnosUs,":[56],"and":[57,94],"resulting":[59],"crowd":[60],"consensus":[61],"labels":[62],"obtained":[63],"by":[64],"aggregating":[65],"using":[66],"plurality":[68],"rule":[69],"significantly":[72],"higher":[73],"78.1%":[76],"(p=0.0014),":[77],"ROC":[80,98],"AUC":[81,99],"(area":[82],"under":[83],"receiver":[85],"operating":[86],"characteristic":[87],"curve)":[88],"0.948":[90],"(95%":[91,102],"CI":[92,103],"0.936-0.959),":[93],"malignant":[95],"versus":[96],"benign":[97],"0.928":[101],"0.911-0.943).":[104],"These":[105],"results":[106],"suggest":[107],"an":[108],"opportunity":[109],"to":[110,114],"harness":[111],"methods":[113],"assist":[115],"creation":[118],"high-quality":[120],"labeled":[121],"datasets":[122],"could":[124],"benefit":[125],"artificial":[127],"intelligence":[128],"(AI)":[129],"development.":[130]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2}],"updated_date":"2026-04-02T15:55:50.835912","created_date":"2025-10-10T00:00:00"}