{"id":"https://openalex.org/W4391368036","doi":"https://doi.org/10.2196/51138","title":"A Perspective on Crowdsourcing and Human-in-the-Loop Workflows in Precision Health","display_name":"A Perspective on Crowdsourcing and Human-in-the-Loop Workflows in Precision Health","publication_year":2024,"publication_date":"2024-01-30","ids":{"openalex":"https://openalex.org/W4391368036","doi":"https://doi.org/10.2196/51138","pmid":"https://pubmed.ncbi.nlm.nih.gov/38602750","pmcid":"https://www.ncbi.nlm.nih.gov/pmc/articles/11046386"},"language":"en","primary_location":{"id":"doi:10.2196/51138","doi":"https://doi.org/10.2196/51138","is_oa":true,"landing_page_url":"https://doi.org/10.2196/51138","pdf_url":null,"source":{"id":"https://openalex.org/S17147534","display_name":"Journal of Medical Internet Research","issn_l":"1438-8871","issn":["1438-8871","1439-4456"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","dongbi-b","doyens","fnege-4","jufo-2","ki-jl-2","medline","norway-2","oarl","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320608","host_organization_name":"JMIR Publications","host_organization_lineage":["https://openalex.org/P4310320608"],"host_organization_lineage_names":["JMIR Publications"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Medical Internet Research","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["arxiv","crossref","datacite","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.2196/51138","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037490912","display_name":"Peter Yigitcan Washington","orcid":"https://orcid.org/0000-0003-3276-4411","observed_orcids":["https://orcid.org/0000-0003-3276-4411"]},"institutions":[{"id":"https://openalex.org/I117965899","display_name":"University of Hawai\u02bbi at M\u0101noa","ror":"https://ror.org/01wspgy28","country_code":"US","type":"education","lineage":["https://openalex.org/I117965899"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Peter Washington","raw_affiliation_strings":["Information and Computer Sciences, University of Hawaii at Manoa, Honolulu, HI, United States"],"raw_orcid":"https://orcid.org/0000-0003-3276-4411","affiliations":[{"raw_affiliation_string":"Information and Computer Sciences, University of Hawaii at Manoa, Honolulu, HI, United States","institution_ids":["https://openalex.org/I117965899"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5037490912"],"corresponding_institution_ids":["https://openalex.org/I117965899"],"apc_list":{"value":2950,"currency":"USD","value_usd":2950},"apc_paid":{"value":2950,"currency":"USD","value_usd":2950},"fwci":3.3161,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":{"value":0.91874867,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"26","issue":null,"first_page":"e51138","last_page":"e51138"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11396","display_name":"Artificial Intelligence in Healthcare","score":0.006399999838322401,"subfield":{"id":"https://openalex.org/subfields/3605","display_name":"Health Information Management"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.0005000000237487257,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"crowdsourcing","score":0.9980000257492065},{"id":"https://openalex.org/keywords/human-in-the-loop-machine-learning","display_name":"human-in-the-loop learning","score":0.9259999990463257},{"id":"https://openalex.org/keywords/precision-health","display_name":"precision health","score":0.9580000042915344},{"id":"https://openalex.org/keywords/diagnostic-devices","display_name":"diagnostic systems","score":0.4830000102519989},{"id":"https://openalex.org/keywords/overfitting","display_name":"overfitting","score":0.8009999990463257},{"id":"https://openalex.org/keywords/gamification","display_name":"gamification","score":0.6759999990463257},{"id":"https://openalex.org/keywords/crowd-annotation","display_name":"crowd annotation","score":0.4230000078678131},{"id":"https://openalex.org/keywords/health-screening","display_name":"health screening","score":0.5329999923706055},{"id":"https://openalex.org/keywords/health-care-access","display_name":"health care access","score":0.8320000171661377}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8921680450439453},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.8179180026054382},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.7377513647079468},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7182029485702515},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7161483764648438},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6530763506889343},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5612990856170654},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.5392564535140991},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5179494619369507},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.4305607080459595},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3458169102668762},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.11515191197395325},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D001523","descriptor_name":"Mental Disorders","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057188","descriptor_name":"Workflow","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057285","descriptor_name":"Precision Medicine","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D063045","descriptor_name":"Crowdsourcing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":7,"locations":[{"id":"doi:10.2196/51138","doi":"https://doi.org/10.2196/51138","is_oa":true,"landing_page_url":"https://doi.org/10.2196/51138","pdf_url":null,"source":{"id":"https://openalex.org/S17147534","display_name":"Journal of Medical Internet Research","issn_l":"1438-8871","issn":["1438-8871","1439-4456"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","dongbi-b","doyens","fnege-4","jufo-2","ki-jl-2","medline","norway-2","oarl","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320608","host_organization_name":"JMIR Publications","host_organization_lineage":["https://openalex.org/P4310320608"],"host_organization_lineage_names":["JMIR Publications"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Medical Internet Research","raw_type":"journal-article"},{"id":"doi:10.2196/preprints.51138","doi":"https://doi.org/10.2196/preprints.51138","is_oa":true,"landing_page_url":"https://doi.org/10.2196/preprints.51138","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},{"id":"pmh:oai:pubmedcentral.nih.gov:11046386","doi":null,"is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/11046386","pdf_url":"https://sistemas-humano-computacionais.wdfiles.com/local--files/capitulo%3Aredes-sociais/Howe_The_Rise_of_Crowdsourcing.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"J Med Internet Res","raw_type":"Text"},{"id":"pmid:38602750","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/38602750","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of medical Internet research","raw_type":"Journal Article"},{"id":"pmh:oai:arXiv.org:2303.03578","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2303.03578","pdf_url":"https://arxiv.org/pdf/2303.03578","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:doaj.org/article:8dbeb4fee5bd4e3baf5645d5c189394b","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/8dbeb4fee5bd4e3baf5645d5c189394b","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of Medical Internet Research, Vol 26, p e51138 (2024)","raw_type":"article"},{"id":"doi:10.48550/arxiv.2303.03578","doi":"https://doi.org/10.48550/arxiv.2303.03578","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2303.03578","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"doi:10.2196/51138","doi":"https://doi.org/10.2196/51138","is_oa":true,"landing_page_url":"https://doi.org/10.2196/51138","pdf_url":null,"source":{"id":"https://openalex.org/S17147534","display_name":"Journal of Medical Internet Research","issn_l":"1438-8871","issn":["1438-8871","1439-4456"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","dongbi-b","doyens","fnege-4","jufo-2","ki-jl-2","medline","norway-2","oarl","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320608","host_organization_name":"JMIR Publications","host_organization_lineage":["https://openalex.org/P4310320608"],"host_organization_lineage_names":["JMIR Publications"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Medical Internet Research","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being","score":0.9391000270843506}],"study_designs":[],"awards":[{"id":"https://openalex.org/G2831263406","display_name":null,"funder_award_id":"DP2 EB035858","funder_id":"https://openalex.org/F4320337363","funder_display_name":"National Institute of Biomedical Imaging and Bioengineering"},{"id":"https://openalex.org/G8758411529","display_name":"Crowd-Powered Machine Learning to Diagnose ASD and ADHD in Adolescents from Digital Social Interactions","funder_award_id":"1dp2eb035858-01","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G9000897584","display_name":"Center for Pacific Innovations, Knowledge, and Opportunities (PIKO)","funder_award_id":"U54GM138062","funder_id":"https://openalex.org/F4320337354","funder_display_name":"National Institute of General Medical Sciences"}],"funders":[{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320337354","display_name":"National Institute of General Medical Sciences","ror":"https://ror.org/04q48ey07"},{"id":"https://openalex.org/F4320337363","display_name":"National Institute of Biomedical Imaging and Bioengineering","ror":"https://ror.org/00372qc85"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391368036.pdf","grobid_xml":"https://content.openalex.org/works/W4391368036.grobid-xml"},"referenced_works_count":90,"referenced_works":["https://openalex.org/W68831505","https://openalex.org/W1520861770","https://openalex.org/W1888870946","https://openalex.org/W1901492489","https://openalex.org/W1937091965","https://openalex.org/W1963761151","https://openalex.org/W1974760327","https://openalex.org/W1979596292","https://openalex.org/W1987308257","https://openalex.org/W2013208127","https://openalex.org/W2014280607","https://openalex.org/W2017107696","https://openalex.org/W2018543616","https://openalex.org/W2029217521","https://openalex.org/W2052006554","https://openalex.org/W2082619040","https://openalex.org/W2086240082","https://openalex.org/W2091360818","https://openalex.org/W2092568776","https://openalex.org/W2098865355","https://openalex.org/W2101798695","https://openalex.org/W2105816875","https://openalex.org/W2106568252","https://openalex.org/W2117539524","https://openalex.org/W2127970560","https://openalex.org/W2135283551","https://openalex.org/W2137489111","https://openalex.org/W2147603330","https://openalex.org/W2149489787","https://openalex.org/W2155871507","https://openalex.org/W2285903333","https://openalex.org/W2339343773","https://openalex.org/W2392412101","https://openalex.org/W2484189903","https://openalex.org/W2584561145","https://openalex.org/W2613081940","https://openalex.org/W2727318663","https://openalex.org/W2735719157","https://openalex.org/W2757470547","https://openalex.org/W2759174152","https://openalex.org/W2773872762","https://openalex.org/W2793617790","https://openalex.org/W2885015237","https://openalex.org/W2894548131","https://openalex.org/W2895763047","https://openalex.org/W2896835644","https://openalex.org/W2897093100","https://openalex.org/W2902092143","https://openalex.org/W2903302073","https://openalex.org/W2905810301","https://openalex.org/W2921813650","https://openalex.org/W2936086693","https://openalex.org/W2938164379","https://openalex.org/W2941097695","https://openalex.org/W2953771225","https://openalex.org/W2963012638","https://openalex.org/W2964397271","https://openalex.org/W2971998890","https://openalex.org/W2972211076","https://openalex.org/W2981093935","https://openalex.org/W2984369117","https://openalex.org/W2985116867","https://openalex.org/W2992144222","https://openalex.org/W3006768956","https://openalex.org/W3008947750","https://openalex.org/W3028078961","https://openalex.org/W3043096756","https://openalex.org/W3048503086","https://openalex.org/W3087864546","https://openalex.org/W3106972798","https://openalex.org/W3108794416","https://openalex.org/W3118357390","https://openalex.org/W3151100342","https://openalex.org/W3164652136","https://openalex.org/W3193149408","https://openalex.org/W3212650557","https://openalex.org/W4205785225","https://openalex.org/W4212799943","https://openalex.org/W4220887861","https://openalex.org/W4221143050","https://openalex.org/W4224252851","https://openalex.org/W4224988111","https://openalex.org/W4226499088","https://openalex.org/W4236058516","https://openalex.org/W4281929536","https://openalex.org/W4282924049","https://openalex.org/W4287505440","https://openalex.org/W4300861274","https://openalex.org/W4317037306","https://openalex.org/W4321437951"],"related_works":["https://openalex.org/W2006726681","https://openalex.org/W2504455947","https://openalex.org/W3106979533","https://openalex.org/W4390246185","https://openalex.org/W2989664921","https://openalex.org/W7048496407","https://openalex.org/W3048503086","https://openalex.org/W4293236878","https://openalex.org/W1607601170","https://openalex.org/W4372049216"],"abstract_inverted_index":{"Modern":[0],"machine":[1,17,139,176],"learning":[2,18,140,177],"approaches":[3],"have":[4],"led":[5],"to":[6,40,93,103,123,136,174,195],"performant":[7],"diagnostic":[8,76,138,159],"models":[9],"for":[10,110,179],"a":[11,43,46,114,125,137,161],"variety":[12],"of":[13,164,172,182],"health":[14,186],"conditions.":[15],"Several":[16],"approaches,":[19],"such":[20],"as":[21,48,134],"decision":[22],"trees":[23],"and":[24,45,61,64,81,151,155,184,192],"deep":[25],"neural":[26],"networks,":[27],"can,":[28],"in":[29,108,147],"principle,":[30],"approximate":[31],"any":[32],"function.":[33],"However,":[34],"this":[35,94,148],"power":[36],"can":[37,73,119,188],"be":[38,41,121],"considered":[39],"both":[42],"gift":[44],"curse,":[47],"the":[49,56,65,132,167,170,180],"propensity":[50],"toward":[51],"overfitting":[52],"is":[53,68,96],"magnified":[54],"when":[55],"input":[57],"data":[58],"are":[59,85,101],"heterogeneous":[60],"high":[62],"dimensional":[63],"output":[66],"class":[67],"highly":[69],"nonlinear.":[70],"This":[71,142],"issue":[72,95],"especially":[74],"plague":[75],"systems":[77],"that":[78,84],"predict":[79],"behavioral":[80,106],"psychiatric":[82],"conditions":[83,187],"diagnosed":[86],"with":[87,157],"subjective":[88],"criteria.":[89],"An":[90],"emerging":[91,149],"solution":[92],"crowdsourcing,":[97],"where":[98],"crowd":[99],"workers":[100],"paid":[102],"annotate":[104],"complex":[105,183],"features":[107],"return":[109],"monetary":[111],"compensation":[112],"or":[113,129],"gamified":[115],"experience.":[116],"These":[117],"labels":[118,133],"then":[120],"used":[122],"derive":[124],"diagnosis,":[126],"either":[127],"directly":[128],"by":[130],"using":[131],"inputs":[135],"model.":[141],"viewpoint":[143],"describes":[144],"existing":[145],"work":[146],"field":[150,163],"discusses":[152],"ongoing":[153],"challenges":[154],"opportunities":[156],"crowd-powered":[158],"systems,":[160],"nascent":[162],"study.":[165],"With":[166],"correct":[168],"considerations,":[169],"addition":[171],"crowdsourcing":[173],"human-in-the-loop":[175],"workflows":[178],"prediction":[181],"nuanced":[185],"accelerate":[189],"screening,":[190],"diagnostics,":[191],"ultimately":[193],"access":[194],"care.":[196]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":6}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2024-01-31T00:00:00"}