{"id":"https://openalex.org/W4386352616","doi":"https://doi.org/10.1109/isbi53787.2023.10230597","title":"Human Not in the Loop: Objective Sample Difficulty Measures for Curriculum Learning","display_name":"Human Not in the Loop: Objective Sample Difficulty Measures for Curriculum Learning","publication_year":2023,"publication_date":"2023-04-18","ids":{"openalex":"https://openalex.org/W4386352616","doi":"https://doi.org/10.1109/isbi53787.2023.10230597","pmid":"https://pubmed.ncbi.nlm.nih.gov/37885672"},"language":"en","primary_location":{"id":"doi:10.1109/isbi53787.2023.10230597","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/isbi53787.2023.10230597","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE 20th International Symposium on Biomedical Imaging (ISBI)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC10602195/pdf/nihms-1891600.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101466459","display_name":"Zhengbo Zhou","orcid":"https://orcid.org/0000-0002-9085-9403"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhengbo Zhou","raw_affiliation_strings":["University of Pittsburgh,Intelligent Systems Program,Pittsburgh,PA,USA","Intelligent Systems Program, University of Pittsburgh, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pittsburgh,Intelligent Systems Program,Pittsburgh,PA,USA","institution_ids":["https://openalex.org/I170201317"]},{"raw_affiliation_string":"Intelligent Systems Program, University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101710818","display_name":"Jun Luo","orcid":"https://orcid.org/0000-0001-9329-7840"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jun Luo","raw_affiliation_strings":["University of Pittsburgh,Intelligent Systems Program,Pittsburgh,PA,USA","Intelligent Systems Program, University of Pittsburgh, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pittsburgh,Intelligent Systems Program,Pittsburgh,PA,USA","institution_ids":["https://openalex.org/I170201317"]},{"raw_affiliation_string":"Intelligent Systems Program, University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074743495","display_name":"Dooman Arefan","orcid":"https://orcid.org/0000-0001-9679-0438"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dooman Arefan","raw_affiliation_strings":["University of Pittsburgh,Department of Radiology,Pittsburgh,PA,USA","Department of Radiology, University of Pittsburgh, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pittsburgh,Department of Radiology,Pittsburgh,PA,USA","institution_ids":["https://openalex.org/I170201317"]},{"raw_affiliation_string":"Department of Radiology, University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061506587","display_name":"Gene Kitamura","orcid":"https://orcid.org/0000-0003-2647-6202"},"institutions":[{"id":"https://openalex.org/I26347476","display_name":"Loma Linda University","ror":"https://ror.org/04bj28v14","country_code":"US","type":"education","lineage":["https://openalex.org/I26347476"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gene Kitamura","raw_affiliation_strings":["Loma Linda University,Department of Radiology,Loma Linda,CA,USA","Department of Radiology, Loma Linda University, Loma Linda, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Loma Linda University,Department of Radiology,Loma Linda,CA,USA","institution_ids":["https://openalex.org/I26347476"]},{"raw_affiliation_string":"Department of Radiology, Loma Linda University, Loma Linda, CA, USA","institution_ids":["https://openalex.org/I26347476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028418236","display_name":"Shandong Wu","orcid":"https://orcid.org/0000-0002-0770-2203"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shandong Wu","raw_affiliation_strings":["University of Pittsburgh,Intelligent Systems Program,Pittsburgh,PA,USA","Department of Biomedical Informatics and Department of Bioengineering, University of Pittsburgh, Pittsburgh, PA, USA","Department of Radiology, University of Pittsburgh, Pittsburgh, PA, USA","Intelligent Systems Program, University of Pittsburgh, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pittsburgh,Intelligent Systems Program,Pittsburgh,PA,USA","institution_ids":["https://openalex.org/I170201317"]},{"raw_affiliation_string":"Department of Biomedical Informatics and Department of Bioengineering, University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]},{"raw_affiliation_string":"Department of Radiology, University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]},{"raw_affiliation_string":"Intelligent Systems Program, University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.3552,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.59473684,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"2023","issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T12574","display_name":"Clinical Reasoning and Diagnostic Skills","score":0.98089998960495,"subfield":{"id":"https://openalex.org/subfields/2714","display_name":"Family Practice"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11775","display_name":"COVID-19 diagnosis using AI","score":0.9114999771118164,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/curriculum","display_name":"Curriculum","score":0.7422559261322021},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.691170334815979},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6212790608406067},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5901616215705872},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5461101531982422},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.5455500483512878},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.517417311668396},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4517662823200226},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.43692517280578613},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.42035698890686035},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12807276844978333},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.11560463905334473},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07912486791610718}],"concepts":[{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.7422559261322021},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.691170334815979},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6212790608406067},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5901616215705872},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5461101531982422},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.5455500483512878},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.517417311668396},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4517662823200226},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.43692517280578613},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.42035698890686035},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12807276844978333},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.11560463905334473},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07912486791610718},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/isbi53787.2023.10230597","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/isbi53787.2023.10230597","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE 20th International Symposium on Biomedical Imaging (ISBI)","raw_type":"proceedings-article"},{"id":"pmid:37885672","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37885672","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings. IEEE International Symposium on Biomedical Imaging","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:10602195","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/10602195","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC10602195/pdf/nihms-1891600.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proc IEEE Int Symp Biomed Imaging","raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:pubmedcentral.nih.gov:10602195","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/10602195","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC10602195/pdf/nihms-1891600.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proc IEEE Int Symp Biomed Imaging","raw_type":"Text"},"sustainable_development_goals":[{"score":0.8799999952316284,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1742171491","display_name":null,"funder_award_id":"1548562","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G2206590661","display_name":null,"funder_award_id":"ACI-1548562","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G282250876","display_name":null,"funder_award_id":"R01CA218405","funder_id":"https://openalex.org/F4320337351","funder_display_name":"National Cancer Institute"},{"id":"https://openalex.org/G3226045097","display_name":null,"funder_award_id":"ACI-1 548 562","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3553923869","display_name":null,"funder_award_id":"R01 EB032896","funder_id":"https://openalex.org/F4320337363","funder_display_name":"National Institute of Biomedical Imaging and Bioengineering"},{"id":"https://openalex.org/G3730782400","display_name":"XSEDE 2.0:  Integrating, Enabling and Enhancing National Cyberinfrastructure with Expanding Community Involvement","funder_award_id":"1548562","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5379480823","display_name":"CICI: SIVD: Discover and defend cyber vulnerabilities of deep learning medical diagnosis models to adversarial attacks","funder_award_id":"2115082","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7404166632","display_name":"Bridges: From Communities and Data to Workflows and Insight","funder_award_id":"1445606","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7657554357","display_name":"Deep interpretation of mammographic images in breast cancer screening","funder_award_id":"5r01ca218405-04","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G7992884429","display_name":null,"funder_award_id":"ACI-1445606","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G8405414688","display_name":"SCH: Leverage clinical knowledge to augment deep learning analysis of breast images","funder_award_id":"5r01eb032896-04","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320310174","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320337351","display_name":"National Cancer Institute","ror":"https://ror.org/040gcmg81"},{"id":"https://openalex.org/F4320337363","display_name":"National Institute of Biomedical Imaging and Bioengineering","ror":"https://ror.org/00372qc85"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386352616.pdf"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W1686810756","https://openalex.org/W2296073425","https://openalex.org/W2605409611","https://openalex.org/W2736374171","https://openalex.org/W2889081631","https://openalex.org/W2962739470","https://openalex.org/W2979643984","https://openalex.org/W3080532707","https://openalex.org/W3103326390","https://openalex.org/W3131581603","https://openalex.org/W3142849873","https://openalex.org/W3203444605","https://openalex.org/W6736518430"],"related_works":["https://openalex.org/W2348562106","https://openalex.org/W2370820329","https://openalex.org/W2370554813","https://openalex.org/W4362576712","https://openalex.org/W2387560707","https://openalex.org/W2363525455","https://openalex.org/W4312355418","https://openalex.org/W2314810092","https://openalex.org/W2384329035","https://openalex.org/W2373380871"],"abstract_inverted_index":{"Curriculum":[0],"learning":[1,4,78],"is":[2,21],"a":[3,10,74,110,144,149],"method":[5,151],"that":[6,152],"trains":[7],"models":[8],"in":[9,116],"meaningful":[11],"order":[12],"from":[13,40,102],"easier":[14],"to":[15,22,43,51,86,112,132,143],"harder":[16],"samples.":[17,30],"A":[18],"key":[19],"here":[20],"devise":[23],"automatic":[24],"and":[25,65,94,159,167,173],"objective":[26,89],"difficulty":[27,47,90],"measures":[28],"of":[29,48,83,92,118],"In":[31,69],"the":[32,81,119,134,140,171],"medical":[33,49],"domain,":[34],"previous":[35,150],"work":[36],"applied":[37],"domain":[38],"knowledge":[39],"human":[41,63,154],"experts":[42],"qualitatively":[44],"assess":[45],"classification":[46,101,120,157,177],"images":[50],"guide":[52,133],"curriculum":[53,77,135,147],"learning,":[54],"which":[55],"requires":[56],"extra":[57],"annotation":[58],"efforts,":[59],"relies":[60],"on":[61,98,156],"subjective":[62],"experience,":[64],"may":[66],"introduce":[67],"bias.":[68],"this":[70],"work,":[71],"we":[72,106],"propose":[73],"new":[75],"automated":[76],"technique":[79,142],"using":[80],"variance":[82],"gradients":[84],"(VoG)":[85],"compute":[87],"an":[88],"measure":[91],"samples":[93],"evaluated":[95],"its":[96],"effects":[97],"elbow":[99],"fracture":[100,176],"X-ray":[103],"images.":[104],"Specifically,":[105],"used":[107,153],"VoG":[108,124],"as":[109],"metric":[111],"rank":[113],"each":[114],"sample":[115],"terms":[117],"difficulty,":[121,158],"where":[122],"high":[123],"scores":[125],"indicate":[126],"more":[127],"difficult":[128],"cases":[129],"for":[130,170],"classification,":[131],"training":[136],"process":[137],"We":[138],"compared":[139],"proposed":[141],"baseline":[145],"(without":[146],"learning),":[148],"annotations":[155],"anti-curriculum":[160],"learning.":[161],"Our":[162],"experiment":[163],"results":[164],"showed":[165],"comparable":[166],"higher":[168],"performance":[169],"binary":[172],"multi-class":[174],"bone":[175],"tasks.":[178]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-08-06T08:24:18.245995","created_date":"2025-10-10T00:00:00"}