{"id":"https://openalex.org/W2805715978","doi":"https://doi.org/10.5281/zenodo.1453313","title":"Is The Winner Really The Best? A Critical Analysis Of Common Research Practice In Biomedical Image Analysis Competitions","display_name":"Is The Winner Really The Best? A Critical Analysis Of Common Research Practice In Biomedical Image Analysis Competitions","publication_year":2018,"publication_date":"2018-06-06","ids":{"openalex":"https://openalex.org/W2805715978","doi":"https://doi.org/10.5281/zenodo.1453313","mag":"2805715978"},"language":"en","primary_location":{"id":"pmh:oai:zenodo.org:1453313","is_oa":true,"landing_page_url":"https://zenodo.org/record/1453313","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/other"},"type":"dataset","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://zenodo.org/record/1453313","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023493127","display_name":"Lena Maier\u2010Hein","orcid":"https://orcid.org/0000-0003-4910-9368"},"institutions":[{"id":"https://openalex.org/I17937529","display_name":"German Cancer Research Center","ror":"https://ror.org/04cdgtt98","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I17937529"]},{"id":"https://openalex.org/I223822909","display_name":"Heidelberg University","ror":"https://ror.org/038t36y30","country_code":"DE","type":"education","lineage":["https://openalex.org/I223822909"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Maier-Hein, Lena","raw_affiliation_strings":["Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany"],"affiliations":[{"raw_affiliation_string":"Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany","institution_ids":["https://openalex.org/I17937529","https://openalex.org/I223822909"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101907663","display_name":"Matthias Eisenmann","orcid":"https://orcid.org/0000-0002-0713-8761"},"institutions":[{"id":"https://openalex.org/I17937529","display_name":"German Cancer Research Center","ror":"https://ror.org/04cdgtt98","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I17937529"]},{"id":"https://openalex.org/I223822909","display_name":"Heidelberg University","ror":"https://ror.org/038t36y30","country_code":"DE","type":"education","lineage":["https://openalex.org/I223822909"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Eisenmann, Matthias","raw_affiliation_strings":["Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany"],"affiliations":[{"raw_affiliation_string":"Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany","institution_ids":["https://openalex.org/I17937529","https://openalex.org/I223822909"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079787198","display_name":"Annika Reinke","orcid":"https://orcid.org/0000-0003-4363-1876"},"institutions":[{"id":"https://openalex.org/I17937529","display_name":"German Cancer Research Center","ror":"https://ror.org/04cdgtt98","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I17937529"]},{"id":"https://openalex.org/I223822909","display_name":"Heidelberg University","ror":"https://ror.org/038t36y30","country_code":"DE","type":"education","lineage":["https://openalex.org/I223822909"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Reinke, Annika","raw_affiliation_strings":["Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany"],"affiliations":[{"raw_affiliation_string":"Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany","institution_ids":["https://openalex.org/I17937529","https://openalex.org/I223822909"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085090260","display_name":"Sinan Onogur","orcid":"https://orcid.org/0000-0002-8371-760X"},"institutions":[{"id":"https://openalex.org/I17937529","display_name":"German Cancer Research Center","ror":"https://ror.org/04cdgtt98","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I17937529"]},{"id":"https://openalex.org/I223822909","display_name":"Heidelberg University","ror":"https://ror.org/038t36y30","country_code":"DE","type":"education","lineage":["https://openalex.org/I223822909"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Onogur, Sinan","raw_affiliation_strings":["Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany"],"affiliations":[{"raw_affiliation_string":"Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany","institution_ids":["https://openalex.org/I17937529","https://openalex.org/I223822909"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080756571","display_name":"Marko Stankovic","orcid":null},"institutions":[{"id":"https://openalex.org/I17937529","display_name":"German Cancer Research Center","ror":"https://ror.org/04cdgtt98","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I17937529"]},{"id":"https://openalex.org/I223822909","display_name":"Heidelberg University","ror":"https://ror.org/038t36y30","country_code":"DE","type":"education","lineage":["https://openalex.org/I223822909"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Stankovic, Marko","raw_affiliation_strings":["Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany"],"affiliations":[{"raw_affiliation_string":"Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany","institution_ids":["https://openalex.org/I17937529","https://openalex.org/I223822909"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112586036","display_name":"Patrick Scholz","orcid":null},"institutions":[{"id":"https://openalex.org/I17937529","display_name":"German Cancer Research Center","ror":"https://ror.org/04cdgtt98","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I17937529"]},{"id":"https://openalex.org/I223822909","display_name":"Heidelberg University","ror":"https://ror.org/038t36y30","country_code":"DE","type":"education","lineage":["https://openalex.org/I223822909"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Scholz, Patrick","raw_affiliation_strings":["Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany"],"affiliations":[{"raw_affiliation_string":"Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany","institution_ids":["https://openalex.org/I17937529","https://openalex.org/I223822909"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5041176403","display_name":"Peter M. Full","orcid":"https://orcid.org/0000-0003-4326-8026"},"institutions":[{"id":"https://openalex.org/I17937529","display_name":"German Cancer Research Center","ror":"https://ror.org/04cdgtt98","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I17937529"]},{"id":"https://openalex.org/I223822909","display_name":"Heidelberg University","ror":"https://ror.org/038t36y30","country_code":"DE","type":"education","lineage":["https://openalex.org/I223822909"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Full, Peter M.","raw_affiliation_strings":["Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany"],"affiliations":[{"raw_affiliation_string":"Division of Computer Assisted Medical Interventions (CAMI), German Cancer Research Center (DKFZ), Heidelberg, Germany","institution_ids":["https://openalex.org/I17937529","https://openalex.org/I223822909"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5023493127"],"corresponding_institution_ids":["https://openalex.org/I17937529","https://openalex.org/I223822909"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12168","display_name":"Health and Medical Research Impacts","score":0.9692000150680542,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T12168","display_name":"Health and Medical Research Impacts","score":0.9692000150680542,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13280","display_name":"Biomedical and Engineering Education","score":0.9376999735832214,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.9147999882698059,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.4566667377948761},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3598576486110687},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3233878016471863},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.32183682918548584}],"concepts":[{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.4566667377948761},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3598576486110687},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3233878016471863},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32183682918548584}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:zenodo.org:1453313","is_oa":true,"landing_page_url":"https://zenodo.org/record/1453313","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/other"},{"id":"doi:10.5281/zenodo.1453313","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.1453313","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"dataset"}],"best_oa_location":{"id":"pmh:oai:zenodo.org:1453313","is_oa":true,"landing_page_url":"https://zenodo.org/record/1453313","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/other"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2478288626","https://openalex.org/W4391913857","https://openalex.org/W2350741829","https://openalex.org/W2530322880"],"abstract_inverted_index":{"This":[0,223],"data":[1,36,331],"set":[2,37],"corresponds":[3],"to":[4,34,58,77,103,121,133,256,313,339,344],"the":[5,8,11,42,47,137,141,161,210,218,238,250,289,295,314,319,328,438,441],"paper:":[6],"Is":[7,437],"winner":[9,439],"really":[10,440],"best?":[12,442],"A":[13,443],"critical":[14,444],"analysis":[15,23,52,126,146,336,445,453],"of":[16,44,49,82,88,112,157,163,205,227,249,297,446],"common":[17,74,96,100,341,447],"research":[18,31,117,448],"practice":[19,75,92,101,342,449],"in":[20,160,207,243,450],"biomedical":[21,50,124,144,164,451],"image":[22,51,125,145,165,452],"competitions":[24],"[1]":[25,244,349],"(Experiment:":[26],"Comprehensive":[27],"reporting).":[28],"The":[29,202,279],"key":[30],"questions":[32],"corresponding":[33],"this":[35],"were:":[38],"RQ1:":[39],"What":[40,72],"is":[41,73],"role":[43],"challenges":[45,56,127,229,259],"for":[46,107,184,236,323],"field":[48,162],"(e.g.":[53,80],"How":[54],"many":[55],"conducted":[57,131],"date?":[59],"In":[60,285],"which":[61,64,69],"fields?":[62],"For":[63],"algorithm":[65],"categories?":[66],"Based":[67,326],"on":[68,178,198,294,327],"modalities?)":[70],"RQ2:":[71],"related":[76,102,343],"challenge":[78,104,239,270,330,345],"design":[79,346],"choice":[81],"metric(s)":[83],"and":[84,109,151,171,186,217,253,306,347],"ranking":[85],"methods,":[86],"number":[87],"training/test":[89],"images,":[90],"annotation":[91],"etc.)?":[93],"Are":[94],"there":[95],"standards?":[97],"RQ3:":[98],"Does":[99],"reporting":[105],"allow":[106],"reproducibility":[108],"adequate":[110],"interpretation":[111],"results?":[113],"To":[114,135],"address":[115],"these":[116],"questions,":[118],"we":[119,139],"aimed":[120],"capture":[122],"all":[123,258],"that":[128,260],"have":[129],"been":[130],"up":[132],"2016.":[134],"acquire":[136],"data,":[138],"analyzed":[140],"websites":[142,156],"hosting/representing":[143],"challenges,":[147],"namely":[148,167],"grand-challenge.org,":[149],"dreamchallenges.org":[150],"kaggle.com":[152],"as":[153,155,265],"well":[154],"main":[158],"conferences":[159],"analysis,":[166],"Medical":[168,189],"Image":[169],"Computing":[170],"Computer":[172],"Assisted":[173],"Intervention":[174],"(MICCAI),":[175],"International":[176,182,196],"Symposium":[177,215],"Biomedical":[179],"Imaging":[180],"(ISBI),":[181],"Society":[183],"Optics":[185],"Photonics":[187],"(SPIE)":[188],"Imaging,":[190],"Cross":[191],"Language":[192],"Evaluation":[193],"Forum":[194],"(CLEF),":[195],"Conference":[197,221],"Pattern":[199],"Recognition":[200],"(ICPR),":[201],"American":[203],"Association":[204],"Physicists":[206],"Medicine":[208],"(AAPM),":[209],"Single":[211],"Molecule":[212],"Localization":[213],"Microscopy":[214],"(SMLMS)":[216],"BioImage":[219],"Informatics":[220],"(BII).":[222],"yielded":[224],"a":[225,234,298,301,307,333],"list":[226,241,316],"150":[228],"with":[230],"549":[231],"tasks.":[232],"Next,":[233],"tool":[235],"instantiating":[237],"parameter":[240,299,315],"introduced":[242],"was":[245,271,304,309,321,337],"used":[246],"by":[247,274],"some":[248],"authors":[251],"(engineers":[252],"medical":[254],"student)":[255],"formalize":[257],"met":[261],"our":[262],"inclusion":[263],"criteria":[264],"follows:":[266],"(1)":[267],"Initially,":[268],"each":[269],"independently":[272],"formalized":[273,329],"two":[275],"different":[276],"observers.":[277],"(2)":[278],"formalization":[280],"results":[281],"were":[282,317],"automatically":[283],"compared.":[284],"ambiguous":[286],"cases,":[287],"when":[288],"observers":[290],"could":[291],"not":[292],"agree":[293],"instantiation":[296],"-":[300],"third":[302],"observer":[303],"consulted,":[305],"decision":[308],"made.":[310],"When":[311],"refinements":[312],"made,":[318],"process":[320],"repeated":[322],"missing":[324],"values.":[325],"set,":[332],"descriptive":[334],"statistical":[335],"performed":[338],"characterize":[340],"reporting.":[348],"Maier-Hein,":[350,395],"L.,":[351],"Eisenmann,":[352],"M.,":[353,359,378,387],"Reinke,":[354],"A.,":[355,370,383,390,422],"Onogur,":[356],"S.,":[357,415],"Stankovic,":[358],"Scholz,":[360],"P.,":[361,368,434],"Arbel,":[362],"T.,":[363],"Bogunovic,":[364],"H.,":[365,399,401],"Bradley,":[366],"A.":[367,374],"Carass,":[369],"Feldmann,":[371],"C.,":[372,411,417],"Frangi,":[373],"F.,":[375,404,426],"Full,":[376],"P.":[377,403],"van":[379,423],"Ginneken,":[380],"B.,":[381],"Hanbury,":[382],"Honauer,":[384],"K.,":[385,392,396,413],"Kozubek,":[386],"Landman,":[388],"B.":[389,398],"M\u00e4rz,":[391],"Maier,":[393],"O.,":[394],"Menze,":[397],"M\u00fcller,":[400],"Neher,":[402],"Niessen,":[405],"W.,":[406],"Rajpoot,":[407],"N.,":[408],"Sharp,":[409],"G.":[410],"Sirinukunwattana,":[412],"Speidel,":[414],"Stock,":[416],"Stoyanov,":[418],"D.,":[419],"Aziz":[420],"Taha,":[421],"der":[424],"Sommen,":[425],"Wang,":[427],"C.-W.,":[428],"Weber,":[429],"M.-A.,":[430],"Zheng,":[431],"G.,":[432],"Jannin,":[433],"Kopp-Schneider,":[435],"A.:":[436],"competitions.":[454],"arXiv":[455],"preprint":[456],"arXiv:1806.02051":[457],"(2018).":[458]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":4}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}