{"id":"https://openalex.org/W4386352616","doi":"https://doi.org/10.1109/isbi53787.2023.10230597","title":"Human Not in the Loop: Objective Sample Difficulty Measures for Curriculum Learning","display_name":"Human Not in the Loop: Objective Sample Difficulty Measures for Curriculum Learning","publication_year":2023,"publication_date":"2023-04-18","ids":{"openalex":"https://openalex.org/W4386352616","doi":"https://doi.org/10.1109/isbi53787.2023.10230597","pmid":"https://pubmed.ncbi.nlm.nih.gov/37885672"},"language":"en","primary_location":{"id":"doi:10.1109/isbi53787.2023.10230597","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/isbi53787.2023.10230597","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE 20th International Symposium on Biomedical Imaging (ISBI)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC10602195/pdf/nihms-1891600.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101466459","display_name":"Zhengbo Zhou","orcid":"https://orcid.org/0000-0002-9085-9403"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhengbo Zhou","raw_affiliation_strings":["University of Pittsburgh,Intelligent Systems Program,Pittsburgh,PA,USA","Intelligent Systems Program, University of Pittsburgh, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pittsburgh,Intelligent Systems Program,Pittsburgh,PA,USA","institution_ids":["https://openalex.org/I170201317"]},{"raw_affiliation_string":"Intelligent Systems Program, University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101710818","display_name":"Jun Luo","orcid":"https://orcid.org/0000-0001-9329-7840"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jun Luo","raw_affiliation_strings":["University of Pittsburgh,Intelligent Systems Program,Pittsburgh,PA,USA","Intelligent Systems Program, University of Pittsburgh, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pittsburgh,Intelligent Systems Program,Pittsburgh,PA,USA","institution_ids":["https://openalex.org/I170201317"]},{"raw_affiliation_string":"Intelligent Systems Program, University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074743495","display_name":"Dooman Arefan","orcid":"https://orcid.org/0000-0001-9679-0438"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dooman Arefan","raw_affiliation_strings":["University of Pittsburgh,Department of Radiology,Pittsburgh,PA,USA","Department of Radiology, University of Pittsburgh, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pittsburgh,Department of Radiology,Pittsburgh,PA,USA","institution_ids":["https://openalex.org/I170201317"]},{"raw_affiliation_string":"Department of Radiology, University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061506587","display_name":"Gene Kitamura","orcid":"https://orcid.org/0000-0003-2647-6202"},"institutions":[{"id":"https://openalex.org/I26347476","display_name":"Loma Linda University","ror":"https://ror.org/04bj28v14","country_code":"US","type":"education","lineage":["https://openalex.org/I26347476"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gene Kitamura","raw_affiliation_strings":["Loma Linda University,Department of Radiology,Loma Linda,CA,USA","Department of Radiology, Loma Linda University, Loma Linda, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Loma Linda University,Department of Radiology,Loma Linda,CA,USA","institution_ids":["https://openalex.org/I26347476"]},{"raw_affiliation_string":"Department of Radiology, Loma Linda University, Loma Linda, CA, USA","institution_ids":["https://openalex.org/I26347476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028418236","display_name":"Shandong Wu","orcid":"https://orcid.org/0000-0002-0770-2203"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shandong Wu","raw_affiliation_strings":["University of Pittsburgh,Intelligent Systems Program,Pittsburgh,PA,USA","Department of Radiology, University of Pittsburgh, Pittsburgh, PA, USA","Intelligent Systems Program, University of Pittsburgh, Pittsburgh, PA, USA","Department of Biomedical Informatics and Department of Bioengineering, University of Pittsburgh, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pittsburgh,Intelligent Systems Program,Pittsburgh,PA,USA","institution_ids":["https://openalex.org/I170201317"]},{"raw_affiliation_string":"Department of Radiology, University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]},{"raw_affiliation_string":"Intelligent Systems Program, University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]},{"raw_affiliation_string":"Department of Biomedical Informatics and Department of Bioengineering, University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.1073,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.53919284,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"2023","issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T12574","display_name":"Clinical Reasoning and Diagnostic Skills","score":0.98089998960495,"subfield":{"id":"https://openalex.org/subfields/2714","display_name":"Family Practice"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11775","display_name":"COVID-19 diagnosis using AI","score":0.9114999771118164,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/curriculum","display_name":"Curriculum","score":0.7422559261322021},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.691170334815979},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6212790608406067},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5901616215705872},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5461101531982422},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.5455500483512878},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.517417311668396},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4517662823200226},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.43692517280578613},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.42035698890686035},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12807276844978333},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.11560463905334473},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07912486791610718}],"concepts":[{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.7422559261322021},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.691170334815979},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6212790608406067},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5901616215705872},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5461101531982422},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.5455500483512878},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.517417311668396},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4517662823200226},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.43692517280578613},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.42035698890686035},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12807276844978333},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.11560463905334473},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07912486791610718},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/isbi53787.2023.10230597","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/isbi53787.2023.10230597","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE 20th International Symposium on Biomedical Imaging (ISBI)","raw_type":"proceedings-article"},{"id":"pmid:37885672","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37885672","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings. IEEE International Symposium on Biomedical Imaging","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:10602195","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/10602195","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC10602195/pdf/nihms-1891600.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proc IEEE Int Symp Biomed Imaging","raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:pubmedcentral.nih.gov:10602195","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/10602195","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC10602195/pdf/nihms-1891600.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proc IEEE Int Symp Biomed Imaging","raw_type":"Text"},"sustainable_development_goals":[{"score":0.8799999952316284,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G1742171491","display_name":null,"funder_award_id":"1548562","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G2158391582","display_name":null,"funder_award_id":"1R01CA218405","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G2206590661","display_name":null,"funder_award_id":"ACI-1548562","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G367708065","display_name":null,"funder_award_id":"ACI-1548562","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3709464396","display_name":null,"funder_award_id":"-1548562","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3730782400","display_name":null,"funder_award_id":"1548562","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4952462017","display_name":null,"funder_award_id":"1R01EB032896","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G5379480823","display_name":null,"funder_award_id":"2115082","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7404166632","display_name":"Bridges: From Communities and Data to Workflows and Insight","funder_award_id":"1445606","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7992884429","display_name":null,"funder_award_id":"ACI-1445606","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G8729765517","display_name":null,"funder_award_id":"ACI-1445606","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8907196067","display_name":null,"funder_award_id":"-1548562","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320310174","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4386352616.pdf"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W1686810756","https://openalex.org/W2296073425","https://openalex.org/W2605409611","https://openalex.org/W2736374171","https://openalex.org/W2889081631","https://openalex.org/W2962739470","https://openalex.org/W2979643984","https://openalex.org/W3080532707","https://openalex.org/W3103326390","https://openalex.org/W3131581603","https://openalex.org/W3142849873","https://openalex.org/W3203444605","https://openalex.org/W6736518430"],"related_works":["https://openalex.org/W2348562106","https://openalex.org/W2370820329","https://openalex.org/W2370554813","https://openalex.org/W4362576712","https://openalex.org/W2387560707","https://openalex.org/W2363525455","https://openalex.org/W4312355418","https://openalex.org/W2314810092","https://openalex.org/W2384329035","https://openalex.org/W2373380871"],"abstract_inverted_index":{"Curriculum":[0],"learning":[1,4,78],"is":[2,21],"a":[3,10,74,110,144,149],"method":[5,151],"that":[6,152],"trains":[7],"models":[8],"in":[9,116],"meaningful":[11],"order":[12],"from":[13,40,102],"easier":[14],"to":[15,22,43,51,86,112,132,143],"harder":[16],"samples.":[17,30],"A":[18],"key":[19],"here":[20],"devise":[23],"automatic":[24],"and":[25,65,94,159,167,173],"objective":[26,89],"difficulty":[27,47,90],"measures":[28],"of":[29,48,83,92,118],"In":[31,69],"the":[32,81,119,134,140,171],"medical":[33,49],"domain,":[34],"previous":[35,150],"work":[36],"applied":[37],"domain":[38],"knowledge":[39],"human":[41,63,154],"experts":[42],"qualitatively":[44],"assess":[45],"classification":[46,101,120,157,177],"images":[50],"guide":[52,133],"curriculum":[53,77,135,147],"learning,":[54],"which":[55],"requires":[56],"extra":[57],"annotation":[58],"efforts,":[59],"relies":[60],"on":[61,98,156],"subjective":[62],"experience,":[64],"may":[66],"introduce":[67],"bias.":[68],"this":[70],"work,":[71],"we":[72,106],"propose":[73],"new":[75],"automated":[76],"technique":[79,142],"using":[80],"variance":[82],"gradients":[84],"(VoG)":[85],"compute":[87],"an":[88],"measure":[91],"samples":[93],"evaluated":[95],"its":[96],"effects":[97],"elbow":[99],"fracture":[100,176],"X-ray":[103],"images.":[104],"Specifically,":[105],"used":[107,153],"VoG":[108,124],"as":[109],"metric":[111],"rank":[113],"each":[114],"sample":[115],"terms":[117],"difficulty,":[121,158],"where":[122],"high":[123],"scores":[125],"indicate":[126],"more":[127],"difficult":[128],"cases":[129],"for":[130,170],"classification,":[131],"training":[136],"process":[137],"We":[138],"compared":[139],"proposed":[141],"baseline":[145],"(without":[146],"learning),":[148],"annotations":[155],"anti-curriculum":[160],"learning.":[161],"Our":[162],"experiment":[163],"results":[164],"showed":[165],"comparable":[166],"higher":[168],"performance":[169],"binary":[172],"multi-class":[174],"bone":[175],"tasks.":[178]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}