{"id":"https://openalex.org/W4387623895","doi":"https://doi.org/10.1109/jbhi.2023.3323582","title":"The Latent Doctor Model for Modeling Inter-Observer Variability","display_name":"The Latent Doctor Model for Modeling Inter-Observer Variability","publication_year":2023,"publication_date":"2023-10-13","ids":{"openalex":"https://openalex.org/W4387623895","doi":"https://doi.org/10.1109/jbhi.2023.3323582","pmid":"https://pubmed.ncbi.nlm.nih.gov/37831571"},"language":"en","primary_location":{"id":"doi:10.1109/jbhi.2023.3323582","is_oa":true,"landing_page_url":"https://doi.org/10.1109/jbhi.2023.3323582","pdf_url":"https://ieeexplore.ieee.org/ielx7/6221020/6363502/10284993.pdf","source":{"id":"https://openalex.org/S2495854775","display_name":"IEEE Journal of Biomedical and Health Informatics","issn_l":"2168-2194","issn":["2168-2194","2168-2208"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Biomedical and Health Informatics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://ieeexplore.ieee.org/ielx7/6221020/6363502/10284993.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081909434","display_name":"Jasper Linmans","orcid":"https://orcid.org/0000-0003-4473-5749"},"institutions":[{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]},{"id":"https://openalex.org/I2802934949","display_name":"Radboud University Medical Center","ror":"https://ror.org/05wg1m734","country_code":"NL","type":"funder","lineage":["https://openalex.org/I2802934949"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Jasper Linmans","raw_affiliation_strings":["Computational Pathology Group, Department of Pathology, Radboud Institute for Health Sciences, Radboud University Medical Center, Nijmegen, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Computational Pathology Group, Department of Pathology, Radboud Institute for Health Sciences, Radboud University Medical Center, Nijmegen, The Netherlands","institution_ids":["https://openalex.org/I145872427","https://openalex.org/I2802934949"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016363084","display_name":"Emiel Hoogeboom","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Emiel Hoogeboom","raw_affiliation_strings":["Independent researcher, Amsterdam, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Independent researcher, Amsterdam, The Netherlands","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081003438","display_name":"Jeroen van der Laak","orcid":"https://orcid.org/0000-0001-7982-0754"},"institutions":[{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]},{"id":"https://openalex.org/I2802934949","display_name":"Radboud University Medical Center","ror":"https://ror.org/05wg1m734","country_code":"NL","type":"funder","lineage":["https://openalex.org/I2802934949"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Jeroen van der Laak","raw_affiliation_strings":["Computational Pathology Group, Department of Pathology, Radboud Institute for Health Sciences, Radboud University Medical Center, Nijmegen, The Netherlands","Chief Scientific Officer (CSO) and shareholder of Aiosyn BV, the Netherlands"],"affiliations":[{"raw_affiliation_string":"Computational Pathology Group, Department of Pathology, Radboud Institute for Health Sciences, Radboud University Medical Center, Nijmegen, The Netherlands","institution_ids":["https://openalex.org/I145872427","https://openalex.org/I2802934949"]},{"raw_affiliation_string":"Chief Scientific Officer (CSO) and shareholder of Aiosyn BV, the Netherlands","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5055697525","display_name":"Geert Litjens","orcid":"https://orcid.org/0000-0003-1554-1291"},"institutions":[{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]},{"id":"https://openalex.org/I2802934949","display_name":"Radboud University Medical Center","ror":"https://ror.org/05wg1m734","country_code":"NL","type":"funder","lineage":["https://openalex.org/I2802934949"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Geert Litjens","raw_affiliation_strings":["Computational Pathology Group, Department of Pathology, Radboud Institute for Health Sciences, Radboud University Medical Center, Nijmegen, The Netherlands","shareholder of Aiosyn BV, the Netherlands"],"affiliations":[{"raw_affiliation_string":"Computational Pathology Group, Department of Pathology, Radboud Institute for Health Sciences, Radboud University Medical Center, Nijmegen, The Netherlands","institution_ids":["https://openalex.org/I145872427","https://openalex.org/I2802934949"]},{"raw_affiliation_string":"shareholder of Aiosyn BV, the Netherlands","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5081909434"],"corresponding_institution_ids":["https://openalex.org/I145872427","https://openalex.org/I2802934949"],"apc_list":null,"apc_paid":null,"fwci":0.69,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.76259225,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"28","issue":"1","first_page":"343","last_page":"354"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10862","display_name":"AI in cancer detection","score":0.9869999885559082,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10862","display_name":"AI in cancer detection","score":0.9869999885559082,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9799000024795532,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.9624999761581421,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7420332431793213},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6120486259460449},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.5402725338935852},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4979720115661621},{"id":"https://openalex.org/keywords/probability-distribution","display_name":"Probability distribution","score":0.4447147846221924},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.4332917034626007},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.42650681734085083},{"id":"https://openalex.org/keywords/observer","display_name":"Observer (physics)","score":0.41883641481399536},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.36460059881210327},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3233732283115387},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.19833391904830933},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1321835219860077}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7420332431793213},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6120486259460449},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.5402725338935852},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4979720115661621},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.4447147846221924},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.4332917034626007},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.42650681734085083},{"id":"https://openalex.org/C2780704645","wikidata":"https://www.wikidata.org/wiki/Q9251458","display_name":"Observer (physics)","level":2,"score":0.41883641481399536},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36460059881210327},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3233732283115387},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.19833391904830933},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1321835219860077},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/jbhi.2023.3323582","is_oa":true,"landing_page_url":"https://doi.org/10.1109/jbhi.2023.3323582","pdf_url":"https://ieeexplore.ieee.org/ielx7/6221020/6363502/10284993.pdf","source":{"id":"https://openalex.org/S2495854775","display_name":"IEEE Journal of Biomedical and Health Informatics","issn_l":"2168-2194","issn":["2168-2194","2168-2208"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Biomedical and Health Informatics","raw_type":"journal-article"},{"id":"pmid:37831571","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37831571","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE journal of biomedical and health informatics","raw_type":null},{"id":"pmh:oai:repository.ubn.ru.nl:2066/303193","is_oa":true,"landing_page_url":"https://hdl.handle.net/2066/303193","pdf_url":"https://repository.ubn.ru.nl//bitstream/handle/2066/303193/303193.pdf","source":{"id":"https://openalex.org/S4306401067","display_name":"Radboud Repository (Radboud University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I145872427","host_organization_name":"Radboud University Nijmegen","host_organization_lineage":["https://openalex.org/I145872427"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article / Letter to editor"},{"id":"pmh:oai:DiVA.org:liu-201009","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:liu:diva-201009","pdf_url":null,"source":{"id":"https://openalex.org/S4306401559","display_name":"KTH Publication Database DiVA (KTH Royal Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1109/jbhi.2023.3323582","is_oa":true,"landing_page_url":"https://doi.org/10.1109/jbhi.2023.3323582","pdf_url":"https://ieeexplore.ieee.org/ielx7/6221020/6363502/10284993.pdf","source":{"id":"https://openalex.org/S2495854775","display_name":"IEEE Journal of Biomedical and Health Informatics","issn_l":"2168-2194","issn":["2168-2194","2168-2208"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Biomedical and Health Informatics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G208971776","display_name":"Role of Nano-Titanium Dioxide Immunotoxicity in Infectious Disease paThology","funder_award_id":"618152","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G4956428346","display_name":null,"funder_award_id":"Horizon 2020 research and innovatio","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5180414528","display_name":null,"funder_award_id":"Innovative Medicines Initiative 2 Joint Undertakin","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5365867299","display_name":null,"funder_award_id":"Grant","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"},{"id":"https://openalex.org/G629491556","display_name":null,"funder_award_id":"(NWO)","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"},{"id":"https://openalex.org/G8051717526","display_name":null,"funder_award_id":"Grant","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G8318064016","display_name":null,"funder_award_id":"Horizon","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G8351690632","display_name":null,"funder_award_id":"945358","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320314178","display_name":"European Federation of Pharmaceutical Industries and Associations","ror":"https://ror.org/00g1x4v36"},{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"},{"id":"https://openalex.org/F4320323698","display_name":"Radboud Universitair Medisch Centrum","ror":"https://ror.org/05wg1m734"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387623895.pdf","grobid_xml":"https://content.openalex.org/works/W4387623895.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W1545430154","https://openalex.org/W1806046400","https://openalex.org/W1901129140","https://openalex.org/W1998118468","https://openalex.org/W2148347694","https://openalex.org/W2332913015","https://openalex.org/W2792474510","https://openalex.org/W2892938835","https://openalex.org/W2921197949","https://openalex.org/W2963446712","https://openalex.org/W2979302305","https://openalex.org/W2979410457","https://openalex.org/W2979901318","https://openalex.org/W2980154109","https://openalex.org/W2995811766","https://openalex.org/W2999091210","https://openalex.org/W3034368386","https://openalex.org/W3036586801","https://openalex.org/W3047561574","https://openalex.org/W3122459568","https://openalex.org/W3138154797","https://openalex.org/W3209194978","https://openalex.org/W4205900565","https://openalex.org/W4207066074","https://openalex.org/W4288346058","https://openalex.org/W6638523607","https://openalex.org/W6640963894","https://openalex.org/W6687045409","https://openalex.org/W6730042731","https://openalex.org/W6744580074","https://openalex.org/W6744627333","https://openalex.org/W6747701563","https://openalex.org/W6752558437","https://openalex.org/W6763099201","https://openalex.org/W6776560590"],"related_works":["https://openalex.org/W2047973478","https://openalex.org/W4295532600","https://openalex.org/W2063823869","https://openalex.org/W2067569035","https://openalex.org/W2090985514","https://openalex.org/W4390516098","https://openalex.org/W2181948922","https://openalex.org/W2384362569","https://openalex.org/W3192036299","https://openalex.org/W4205302943"],"abstract_inverted_index":{"Many":[0],"inherently":[1],"ambiguous":[2],"tasks":[3],"in":[4,12,37,172],"medical":[5,38],"imaging":[6],"suffer":[7],"from":[8],"inter-observer":[9],"variability,":[10],"resulting":[11],"a":[13,18,28,46,67,82,113,127,194],"reference":[14],"standard":[15],"defined":[16],"by":[17],"distribution":[19,61,128,142,167],"of":[20,48,69,125,129,183],"labels":[21],"with":[22,112,186],"high":[23],"variance.":[24],"Training":[25],"only":[26],"on":[27,43,57,88,116,193],"consensus":[29],"or":[30],"majority":[31,118,147],"vote":[32,119,148],"label,":[33],"as":[34,97],"is":[35,136],"common":[36],"imaging,":[39],"discards":[40],"valuable":[41],"information":[42],"uncertainty":[44,65,171],"amongst":[45],"panel":[47,68],"experts.":[49],"In":[50,103],"this":[51],"work,":[52],"we":[53,80,94,108,156,179],"propose":[54,81],"to":[55,62,96,138,151],"train":[56],"the":[58,64,72,89,98,110,117,134,140,146,152,159,165,184,187],"full":[59],"label":[60,120,166],"predict":[63],"within":[66],"experts":[70],"and":[71,121,168],"most":[73],"likely":[74],"ground-truth":[75],"label.":[76],"To":[77],"do":[78],"so,":[79],"new":[83],"stochastic":[84],"classification":[85,197],"framework":[86],"based":[87],"conditional":[90],"variational":[91],"auto-encoder,":[92],"which":[93],"refer":[95],"Latent":[99],"Doctor":[100],"Model":[101],"(LDM).":[102],"an":[104],"extensive":[105],"comparative":[106],"analysis,":[107],"compare":[109],"LDM":[111,135,160,185],"model":[114],"trained":[115],"other":[122,153],"methods":[123],"capable":[124],"learning":[126],"labels.":[130],"We":[131],"show":[132,180],"that":[133,158],"able":[137],"reproduce":[139],"reference-standard":[141],"significantly":[143],"better":[144],"than":[145],"baseline.":[149],"Compared":[150],"baseline":[154],"methods,":[155],"demonstrate":[157],"performs":[161],"best":[162],"at":[163],"modeling":[164],"its":[169],"corresponding":[170],"two":[173],"prostate":[174],"tumor":[175,195],"grading":[176],"tasks.":[177],"Furthermore,":[178],"competitive":[181],"performance":[182],"more":[188],"computationally":[189],"demanding":[190],"deep":[191],"ensembles":[192],"budding":[196],"task.":[198]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}