{"id":"https://openalex.org/W3046830682","doi":"https://doi.org/10.48550/arxiv.2007.15963","title":"Disentangling Human Error from the Ground Truth in Segmentation of Medical Images","display_name":"Disentangling Human Error from the Ground Truth in Segmentation of Medical Images","publication_year":2020,"publication_date":"2020-07-31","ids":{"openalex":"https://openalex.org/W3046830682","doi":"https://doi.org/10.48550/arxiv.2007.15963","mag":"3046830682"},"language":"en","primary_location":{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/82912999-2944-4d68-a7f2-3f9315247801","doi":null,"is_oa":false,"landing_page_url":"https://research.vumc.nl/en/publications/82912999-2944-4d68-a7f2-3f9315247801","pdf_url":null,"source":{"id":"https://openalex.org/S7407055350","display_name":"Amsterdam UMC (VU Amsterdam) - Institutional Repository","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Zhang, L, Tanno, R, Xu, M-C, Jin, C, Jacob, J, Ciccarelli, O, Barkhof, F & Alexander, D C 2020, Disentangling human error from the ground truth in segmentation of medical images. in 34th Conference on Neural Information Processing Systems, NeurIPS 2020. vol. 2020-December, Advances in Neural Information Processing Systems, Neural information processing systems foundation, 34th Conference on Neural Information Processing Systems, NeurIPS 2020, Virtual, Online, 06/12/2020.","raw_type":"info:eu-repo/semantics/conferenceObject"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2007.15963","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100775917","display_name":"Le Zhang","orcid":"https://orcid.org/0000-0002-3848-0017","observed_orcids":["https://orcid.org/0000-0002-3848-0017"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Le","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064157732","display_name":"Ryutaro Tanno","orcid":"https://orcid.org/0000-0002-8107-6730","observed_orcids":["https://orcid.org/0000-0002-8107-6730"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tanno, Ryutaro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073094674","display_name":"Moucheng Xu","orcid":"https://orcid.org/0000-0003-3368-9871","observed_orcids":["https://orcid.org/0000-0003-3368-9871"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Mou-Cheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Jin, Chen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009829456","display_name":"Joseph Jacob","orcid":"https://orcid.org/0000-0002-8054-2293","observed_orcids":["https://orcid.org/0000-0002-8054-2293"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jacob, Joseph","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067383775","display_name":"Olga Ciccarelli","orcid":"https://orcid.org/0000-0001-7485-1367","observed_orcids":["https://orcid.org/0000-0001-7485-1367","https://orcid.org/0000-0003-0638-1660"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ciccarelli, Olga","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064378373","display_name":"Frederik Barkhof","orcid":"https://orcid.org/0000-0003-3543-3706","observed_orcids":["https://orcid.org/0000-0003-3543-3706"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barkhof, Frederik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5033449704","display_name":"Daniel C. Alexander","orcid":"https://orcid.org/0000-0003-2439-350X","observed_orcids":["https://orcid.org/0000-0003-2439-350X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alexander, Daniel C.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.8558,"has_fulltext":false,"cited_by_count":92,"citation_normalized_percentile":{"value":0.94860406,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":100},"biblio":{"volume":"33","issue":null,"first_page":"15750","last_page":"15762"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10052","display_name":"Medical Image Segmentation Techniques","score":0.9883000254631042,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10052","display_name":"Medical Image Segmentation Techniques","score":0.9883000254631042,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.003100000089034438,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.0027000000700354576,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/noisy-labels","display_name":"label noise","score":0.7990000247955322},{"id":"https://openalex.org/keywords/medical-image-segmentation","display_name":"medical image segmentation","score":0.796999990940094},{"id":"https://openalex.org/keywords/interobserver-variability","display_name":"interobserver variability","score":0.8289999961853027},{"id":"https://openalex.org/keywords/multiple-sclerosis-lesion-segmentation","display_name":"multiple sclerosis lesion segmentation","score":0.7269999980926514},{"id":"https://openalex.org/keywords/brain-tumor-segmentation","display_name":"brain tumor segmentation","score":0.9520000219345093},{"id":"https://openalex.org/keywords/pulmonary-nodule-segmentation","display_name":"lung nodule segmentation","score":0.8489999771118164}],"concepts":[{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.8347018361091614},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.6608744263648987},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6316066980361938},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.5515503883361816},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.44863975048065186},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3376540541648865}],"mesh":[],"locations_count":8,"locations":[{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/82912999-2944-4d68-a7f2-3f9315247801","doi":null,"is_oa":false,"landing_page_url":"https://research.vumc.nl/en/publications/82912999-2944-4d68-a7f2-3f9315247801","pdf_url":null,"source":{"id":"https://openalex.org/S7407055350","display_name":"Amsterdam UMC (VU Amsterdam) - Institutional Repository","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Zhang, L, Tanno, R, Xu, M-C, Jin, C, Jacob, J, Ciccarelli, O, Barkhof, F & Alexander, D C 2020, Disentangling human error from the ground truth in segmentation of medical images. in 34th Conference on Neural Information Processing Systems, NeurIPS 2020. vol. 2020-December, Advances in Neural Information Processing Systems, Neural information processing systems foundation, 34th Conference on Neural Information Processing Systems, NeurIPS 2020, Virtual, Online, 06/12/2020.","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:arXiv.org:2007.15963","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2007.15963","pdf_url":"https://arxiv.org/pdf/2007.15963","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:eprints.ucl.ac.uk.OAI2:10108140","doi":null,"is_oa":false,"landing_page_url":"https://discovery.ucl.ac.uk/id/eprint/10108140/","pdf_url":null,"source":{"id":"https://openalex.org/S4306400024","display_name":"UCL Discovery (University College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I45129253","host_organization_name":"University College London","host_organization_lineage":["https://openalex.org/I45129253"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Advances in Neural Information Processing Systems , 2020-D (2020)","raw_type":"Article"},{"id":"pmh:oai:pure.amsterdamumc.nl:openaire_cris_publications/d67ac5f5-777a-4aaf-be01-cddd74060298","doi":null,"is_oa":false,"landing_page_url":"https://www.scopus.com/pages/publications/85103810949","pdf_url":null,"source":{"id":"https://openalex.org/S7407055222","display_name":"Pure Amsterdam UMC","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Zhang, L, Tanno, R, Xu, M-C, Jin, C, Jacob, J, Ciccarelli, O, Barkhof, F & Alexander, D C 2020, Disentangling human error from the ground truth in segmentation of medical images. in 34th Conference on Neural Information Processing Systems, NeurIPS 2020. vol. 2020-December, Advances in Neural Information Processing Systems, Neural information processing systems foundation, 34th Conference on Neural Information Processing Systems, NeurIPS 2020, Virtual, Online, 06/12/2020.","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/c03ff4c6-b631-4691-b80e-67439f0318a3","doi":null,"is_oa":false,"landing_page_url":"https://research.birmingham.ac.uk/en/publications/c03ff4c6-b631-4691-b80e-67439f0318a3","pdf_url":null,"source":{"id":"https://openalex.org/S4306402634","display_name":"University of Birmingham Research Portal (University of Birmingham)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I79619799","host_organization_name":"University of Birmingham","host_organization_lineage":["https://openalex.org/I79619799"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Zhang, L, Tanno, R, Xu, M, Jin, C, Jacob, J, Ciccarelli, O, Barkhof, F & Alexander, D C 2020, Disentangling Human Error from the Ground Truth in Segmentation of Medical Images. in Advances in Neural Information Processing Systems (NeurIPS).","raw_type":"contributionToPeriodical"},{"id":"pmh:vumc:oai:pure.atira.dk:publications/82912999-2944-4d68-a7f2-3f9315247801","doi":null,"is_oa":false,"landing_page_url":"https://research.vumc.nl/en/publications/82912999-2944-4d68-a7f2-3f9315247801","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"34th Conference on Neural Information Processing Systems, NeurIPS 2020, 2020-December","raw_type":"info:eu-repo/semantics/conferencepaper"},{"id":"doi:10.48550/arxiv.2007.15963","doi":"https://doi.org/10.48550/arxiv.2007.15963","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2007.15963","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:3101188145","doi":null,"is_oa":false,"landing_page_url":"https://proceedings.neurips.cc/paper/2020/file/b5d17ed2b502da15aa727af0d51508d6-Paper.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"Neural Information Processing Systems","raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2007.15963","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2007.15963","pdf_url":"https://arxiv.org/pdf/2007.15963","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2810537366","display_name":null,"funder_award_id":"EP/M020533/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G3953918217","display_name":null,"funder_award_id":"EP/R006032/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W1641498739","https://openalex.org/W1858721549","https://openalex.org/W1901129140","https://openalex.org/W1963801017","https://openalex.org/W1980933880","https://openalex.org/W1986649315","https://openalex.org/W2004293194","https://openalex.org/W2010135967","https://openalex.org/W2024050078","https://openalex.org/W2040154627","https://openalex.org/W2085091083","https://openalex.org/W2097036415","https://openalex.org/W2100209835","https://openalex.org/W2107954485","https://openalex.org/W2116895317","https://openalex.org/W2134305421","https://openalex.org/W2135761719","https://openalex.org/W2138328476","https://openalex.org/W2144660879","https://openalex.org/W2148347694","https://openalex.org/W2149273804","https://openalex.org/W2150612552","https://openalex.org/W2154096410","https://openalex.org/W2161913873","https://openalex.org/W2164971516","https://openalex.org/W2167878392","https://openalex.org/W2181558882","https://openalex.org/W2745544219","https://openalex.org/W2772357980","https://openalex.org/W2776638780","https://openalex.org/W2798896881","https://openalex.org/W2849179291","https://openalex.org/W2888366521","https://openalex.org/W2889615630","https://openalex.org/W2890354008","https://openalex.org/W2911546262","https://openalex.org/W2912968987","https://openalex.org/W2918342466","https://openalex.org/W2963632741","https://openalex.org/W2964121744","https://openalex.org/W2979338721","https://openalex.org/W2979410457","https://openalex.org/W2979874173","https://openalex.org/W2997880344"],"related_works":["https://openalex.org/W4320035307","https://openalex.org/W3091120243","https://openalex.org/W4386408865","https://openalex.org/W4402981957","https://openalex.org/W7106290961","https://openalex.org/W3177819996","https://openalex.org/W4409983021","https://openalex.org/W3176735645","https://openalex.org/W7214199791","https://openalex.org/W4319996358"],"abstract_inverted_index":{"Recent":[0],"years":[1],"have":[2],"seen":[3],"increasing":[4],"use":[5],"of":[6,17,24,58,66,108,123,160,169,216,223,239],"supervised":[7],"learning":[8],"methods":[9,206],"for":[10,98],"segmentation":[11,61,87,114,151,177],"tasks.":[12],"However,":[13],"the":[14,22,32,38,59,64,79,83,106,112,124,130,142,158,161,167,170,214,221,235],"predictive":[15],"performance":[16,84],"these":[18,74],"algorithms":[19,88],"depends":[20],"on":[21,154,172],"quality":[23],"labels.":[25],"This":[26],"problem":[27],"is":[28,126,218,225],"particularly":[29,210],"pertinent":[30],"in":[31,211],"medical":[33,175],"image":[34],"domain,":[35],"where":[36,213],"both":[37],"annotation":[39],"cost":[40],"and":[41,70,111,156,183,207,220],"inter-observer":[42],"variability":[43],"are":[44],"high.":[45],"In":[46,91,199],"a":[47,96,149],"typical":[48],"label":[49,115],"acquisition":[50],"process,":[51],"different":[52],"human":[53],"experts":[54],"provide":[55],"their":[56,67],"estimates":[57],"\"true\"":[60],"labels":[62,76],"under":[63],"influence":[65],"own":[68],"biases":[69],"competence":[71],"levels.":[72],"Treating":[73],"noisy":[75,103,143],"blindly":[77],"as":[78],"ground":[80],"truth":[81],"limits":[82],"that":[85],"automatic":[86],"can":[89],"achieve.":[90],"this":[92],"work,":[93],"we":[94],"present":[95],"method":[97,171,203],"jointly":[99],"learning,":[100],"from":[101],"purely":[102],"observations":[104],"alone,":[105],"reliability":[107],"individual":[109],"annotators":[110,132],"true":[113],"distributions,":[116],"using":[117],"two":[118,125],"coupled":[119],"CNNs.":[120],"The":[121,227],"separation":[122],"achieved":[127],"by":[128],"encouraging":[129],"estimated":[131],"to":[133,233],"be":[134],"maximally":[135],"unreliable":[136],"while":[137],"achieving":[138],"high":[139],"fidelity":[140],"with":[141,179],"training":[144],"data.":[145],"We":[146,164],"first":[147],"define":[148],"toy":[150],"dataset":[152],"based":[153],"MNIST":[155],"study":[157],"properties":[159],"proposed":[162],"algorithm.":[163],"then":[165],"demonstrate":[166],"utility":[168],"three":[173],"public":[174],"imaging":[176],"datasets":[178],"simulated":[180],"(when":[181],"necessary)":[182],"real":[184],"diverse":[185],"annotations:":[186],"1)":[187],"MSLSC":[188],"(multiple-sclerosis":[189],"lesions);":[190],"2)":[191],"BraTS":[192],"(brain":[193],"tumours);":[194],"3)":[195],"LIDC-IDRI":[196],"(lung":[197],"abnormalities).":[198],"all":[200],"cases,":[201],"our":[202],"outperforms":[204],"competing":[205],"relevant":[208],"baselines":[209],"cases":[212],"number":[215],"annotations":[217],"small":[219],"amount":[222],"disagreement":[224],"large.":[226],"experiments":[228],"also":[229],"show":[230],"strong":[231],"ability":[232],"capture":[234],"complex":[236],"spatial":[237],"characteristics":[238],"annotators'":[240],"mistakes.":[241]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":28},{"year":2023,"cited_by_count":24},{"year":2022,"cited_by_count":16},{"year":2021,"cited_by_count":10}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2020-08-07T00:00:00"}