{"id":"https://openalex.org/W4413640844","doi":"https://doi.org/10.3390/bdcc9090220","title":"Assessing the Influence of Feedback Strategies on Errors in Crowdsourced Annotation of Tumor Images","display_name":"Assessing the Influence of Feedback Strategies on Errors in Crowdsourced Annotation of Tumor Images","publication_year":2025,"publication_date":"2025-08-26","ids":{"openalex":"https://openalex.org/W4413640844","doi":"https://doi.org/10.3390/bdcc9090220"},"language":"en","primary_location":{"id":"doi:10.3390/bdcc9090220","is_oa":true,"landing_page_url":"https://doi.org/10.3390/bdcc9090220","pdf_url":"https://www.mdpi.com/2504-2289/9/9/220/pdf?version=1756218712","source":{"id":"https://openalex.org/S4210238752","display_name":"Big Data and Cognitive Computing","issn_l":"2504-2289","issn":["2504-2289"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Big Data and Cognitive Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2504-2289/9/9/220/pdf?version=1756218712","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052959283","display_name":"Jose Libreros","orcid":"https://orcid.org/0000-0002-5434-5464"},"institutions":[{"id":"https://openalex.org/I119449181","display_name":"Technische Universit\u00e4t Ilmenau","ror":"https://ror.org/01weqhp73","country_code":"DE","type":"education","lineage":["https://openalex.org/I119449181"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Jose Alejandro Libreros","raw_affiliation_strings":["User-Centric Analysis of Multimedia Data Research Group, Faculty of Electrical Engineering and Information Technology, Technische Universit\u00e4t Ilmenau, Gustav-Kirchhoff-Stra\u00dfe 1, 98693 Ilmenau, Germany"],"affiliations":[{"raw_affiliation_string":"User-Centric Analysis of Multimedia Data Research Group, Faculty of Electrical Engineering and Information Technology, Technische Universit\u00e4t Ilmenau, Gustav-Kirchhoff-Stra\u00dfe 1, 98693 Ilmenau, Germany","institution_ids":["https://openalex.org/I119449181"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058473354","display_name":"Edwin Gamboa","orcid":"https://orcid.org/0000-0002-5037-5279"},"institutions":[{"id":"https://openalex.org/I4210088386","display_name":"Sysmex (Germany)","ror":"https://ror.org/007416t51","country_code":"DE","type":"company","lineage":["https://openalex.org/I4210088386"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Edwin Gamboa","raw_affiliation_strings":["ScaleHub GmbH, Heidbergstra\u00dfe 100, 22846 Norderstedt, Germany"],"affiliations":[{"raw_affiliation_string":"ScaleHub GmbH, Heidbergstra\u00dfe 100, 22846 Norderstedt, Germany","institution_ids":["https://openalex.org/I4210088386"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083281091","display_name":"Erik Henke","orcid":"https://orcid.org/0000-0003-2380-2682"},"institutions":[{"id":"https://openalex.org/I25974101","display_name":"University of W\u00fcrzburg","ror":"https://ror.org/00fbnyb24","country_code":"DE","type":"education","lineage":["https://openalex.org/I25974101"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Erik Henke","raw_affiliation_strings":["Institute of Anatomy and Cell Biology, Faculty of Medicine, Julius-Maximilians-Universit\u00e4t W\u00fcrzburg, Koellikerstra\u00dfe 6, 97070 W\u00fcrzburg, Germany"],"affiliations":[{"raw_affiliation_string":"Institute of Anatomy and Cell Biology, Faculty of Medicine, Julius-Maximilians-Universit\u00e4t W\u00fcrzburg, Koellikerstra\u00dfe 6, 97070 W\u00fcrzburg, Germany","institution_ids":["https://openalex.org/I25974101"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5041286648","display_name":"Matthias Hirth","orcid":"https://orcid.org/0000-0002-1359-363X"},"institutions":[{"id":"https://openalex.org/I119449181","display_name":"Technische Universit\u00e4t Ilmenau","ror":"https://ror.org/01weqhp73","country_code":"DE","type":"education","lineage":["https://openalex.org/I119449181"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Matthias Hirth","raw_affiliation_strings":["User-Centric Analysis of Multimedia Data Research Group, Faculty of Electrical Engineering and Information Technology, Technische Universit\u00e4t Ilmenau, Gustav-Kirchhoff-Stra\u00dfe 1, 98693 Ilmenau, Germany"],"affiliations":[{"raw_affiliation_string":"User-Centric Analysis of Multimedia Data Research Group, Faculty of Electrical Engineering and Information Technology, Technische Universit\u00e4t Ilmenau, Gustav-Kirchhoff-Stra\u00dfe 1, 98693 Ilmenau, Germany","institution_ids":["https://openalex.org/I119449181"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5041286648","https://openalex.org/A5052959283"],"corresponding_institution_ids":["https://openalex.org/I119449181"],"apc_list":{"value":1400,"currency":"CHF","value_usd":1515},"apc_paid":{"value":1400,"currency":"CHF","value_usd":1515},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.40349358,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"9","issue":"9","first_page":"220","last_page":"220"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14254","display_name":"Digital Media and Visual Art","score":0.8718000054359436,"subfield":{"id":"https://openalex.org/subfields/1704","display_name":"Computer Graphics and Computer-Aided Design"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14254","display_name":"Digital Media and Visual Art","score":0.8718000054359436,"subfield":{"id":"https://openalex.org/subfields/1704","display_name":"Computer Graphics and Computer-Aided Design"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12422","display_name":"Radiomics and Machine Learning in Medical Imaging","score":0.847000002861023,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10862","display_name":"AI in cancer detection","score":0.8274000287055969,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.7880287170410156},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5155086517333984},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.508820116519928},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.41498059034347534},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3212195634841919},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.15561428666114807}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7880287170410156},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5155086517333984},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.508820116519928},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41498059034347534},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3212195634841919},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.15561428666114807}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3390/bdcc9090220","is_oa":true,"landing_page_url":"https://doi.org/10.3390/bdcc9090220","pdf_url":"https://www.mdpi.com/2504-2289/9/9/220/pdf?version=1756218712","source":{"id":"https://openalex.org/S4210238752","display_name":"Big Data and Cognitive Computing","issn_l":"2504-2289","issn":["2504-2289"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Big Data and Cognitive Computing","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:a011440ba266488b8568e993bbeb83ea","is_oa":true,"landing_page_url":"https://doaj.org/article/a011440ba266488b8568e993bbeb83ea","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Big Data and Cognitive Computing, Vol 9, Iss 9, p 220 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/bdcc9090220","is_oa":true,"landing_page_url":"https://doi.org/10.3390/bdcc9090220","pdf_url":"https://www.mdpi.com/2504-2289/9/9/220/pdf?version=1756218712","source":{"id":"https://openalex.org/S4210238752","display_name":"Big Data and Cognitive Computing","issn_l":"2504-2289","issn":["2504-2289"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Big Data and Cognitive Computing","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4413640844.pdf","grobid_xml":"https://content.openalex.org/works/W4413640844.grobid-xml"},"referenced_works_count":44,"referenced_works":["https://openalex.org/W1833093522","https://openalex.org/W1923698532","https://openalex.org/W2007615281","https://openalex.org/W2041282815","https://openalex.org/W2059700429","https://openalex.org/W2061554433","https://openalex.org/W2074790166","https://openalex.org/W2168745915","https://openalex.org/W2171913066","https://openalex.org/W2293201672","https://openalex.org/W2335741193","https://openalex.org/W2475282416","https://openalex.org/W2530695086","https://openalex.org/W2611598635","https://openalex.org/W2765774350","https://openalex.org/W2783946743","https://openalex.org/W2898546434","https://openalex.org/W2900512194","https://openalex.org/W2901536645","https://openalex.org/W2902855764","https://openalex.org/W2904508208","https://openalex.org/W2922239620","https://openalex.org/W2961684664","https://openalex.org/W2985155603","https://openalex.org/W3000783465","https://openalex.org/W3006363044","https://openalex.org/W3133342615","https://openalex.org/W3156124933","https://openalex.org/W3159184664","https://openalex.org/W3164897936","https://openalex.org/W3182402394","https://openalex.org/W4205588474","https://openalex.org/W4210252094","https://openalex.org/W4239930956","https://openalex.org/W4242192106","https://openalex.org/W4301970156","https://openalex.org/W4308885736","https://openalex.org/W4319337589","https://openalex.org/W4319832904","https://openalex.org/W4381280738","https://openalex.org/W4387827595","https://openalex.org/W4395007105","https://openalex.org/W4395688945","https://openalex.org/W4401667784"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4387369504","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W3107602296","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"Crowdsourcing":[0],"enables":[1],"the":[2,31,41,47,54,66,112,135,154,159,165,176,186],"acquisition":[3],"of":[4,49,137,188],"distributed":[5],"human":[6,12,26],"intelligence":[7],"for":[8,65,83,105],"solving":[9],"tasks":[10,32,42,51,85,169],"involving":[11],"judgments":[13],"in":[14,21,79,167],"scalable":[15],"ways,":[16],"with":[17],"many":[18],"use":[19],"cases":[20],"various":[22,50],"application":[23],"areas":[24],"accessing":[25],"intelligence.":[27],"However,":[28,75],"crowdworkers":[29,67,77,106,198],"completing":[30],"may":[33],"have":[34],"limited":[35],"or":[36],"no":[37],"background":[38],"knowledge":[39],"about":[40,193],"they":[43],"solve":[44],"due":[45],"to":[46,61,68,71],"plethora":[48],"available.":[52],"Therefore,":[53],"tasks\u2014even":[55],"on":[56,111,127],"a":[57,80,87,120,124],"micro":[58],"scale\u2014also":[59],"need":[60],"include":[62],"appropriate":[63],"training":[64,76,103,122,141,155],"enable":[69],"them":[70,73],"complete":[72],"successfully.":[74],"efficiently":[78],"short":[81],"time":[82],"complex":[84],"poses":[86],"challenge":[88,98],"and":[89,107,134,142,157,170,174],"remains":[90],"an":[91],"unresolved":[92],"issue.":[93],"This":[94],"paper":[95],"addresses":[96],"this":[97],"by":[99,131,196],"empirically":[100],"comparing":[101],"different":[102],"strategies":[104],"evaluating":[108],"their":[109],"impact":[110],"crowdworkers\u2019":[113],"task":[114,143,160],"results.":[115,177],"We":[116,178],"perform":[117],"comparisons":[118],"between":[119],"basic":[121],"strategy,":[123],"strategy":[125],"based":[126],"previous":[128],"errors":[129,173],"made":[130,195],"other":[132,197],"crowdworkers,":[133],"addition":[136],"instant":[138,150,189],"feedback":[139,151,190],"during":[140,152,158],"completion.":[144],"Our":[145],"results":[146],"show":[147],"that":[148,180],"adding":[149],"both":[153],"phase":[156],"yields":[161],"more":[162,181],"attention":[163,182],"from":[164],"workers":[166],"difficult":[168],"hence":[171],"reduces":[172],"improves":[175],"conclude":[179],"is":[183],"retained":[184],"when":[185],"content":[187],"includes":[191],"information":[192],"mistakes":[194],"previously.":[199]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}