{"id":"https://openalex.org/W2156274477","doi":"https://doi.org/10.1148/radiol.2533081632","title":"Experimental Design and Data Analysis in Receiver Operating Characteristic Studies: Lessons Learned from Reports inRadiologyfrom 1997 to 2006","display_name":"Experimental Design and Data Analysis in Receiver Operating Characteristic Studies: Lessons Learned from Reports inRadiologyfrom 1997 to 2006","publication_year":2009,"publication_date":"2009-10-29","ids":{"openalex":"https://openalex.org/W2156274477","doi":"https://doi.org/10.1148/radiol.2533081632","mag":"2156274477","pmid":"https://pubmed.ncbi.nlm.nih.gov/19864510","pmcid":"https://www.ncbi.nlm.nih.gov/pmc/articles/2786192"},"language":"en","primary_location":{"id":"doi:10.1148/radiol.2533081632","doi":"https://doi.org/10.1148/radiol.2533081632","is_oa":false,"landing_page_url":"https://doi.org/10.1148/radiol.2533081632","pdf_url":null,"source":{"id":"https://openalex.org/S50280174","display_name":"Radiology","issn_l":"0033-8419","issn":["0033-8419","1527-1315"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","dongbi-a","doyens","jufo-3","ki-jl-3","medline","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315931","host_organization_name":"Radiological Society of North America","host_organization_lineage":["https://openalex.org/P4310315931"],"host_organization_lineage_names":["Radiological Society of North America"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Radiology","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/2786192","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027216693","display_name":"Junji Shiraishi","orcid":"https://orcid.org/0000-0002-7899-9876","observed_orcids":["https://orcid.org/0000-0002-7899-9876"]},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Junji Shiraishi","raw_affiliation_strings":["Kurt Rossmann Laboratories for Radiologic Image Research, Department of Radiology, University of Chicago, Chicago, IL, USA. j2s@kumamoto-u.ac.jp","1From the Kurt Rossmann Laboratories for Radiologic Image Research, Department of Radiology, University of Chicago, Chicago, IL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kurt Rossmann Laboratories for Radiologic Image Research, Department of Radiology, University of Chicago, Chicago, IL, USA. j2s@kumamoto-u.ac.jp","institution_ids":["https://openalex.org/I40347166"]},{"raw_affiliation_string":"1From the Kurt Rossmann Laboratories for Radiologic Image Research, Department of Radiology, University of Chicago, Chicago, IL","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048354436","display_name":"Lorenzo L. Pesce","orcid":"https://orcid.org/0000-0002-8015-4653","observed_orcids":["https://orcid.org/0000-0002-8015-4653"]},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lorenzo L. Pesce","raw_affiliation_strings":["1From the Kurt Rossmann Laboratories for Radiologic Image Research, Department of Radiology, University of Chicago, Chicago, IL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"1From the Kurt Rossmann Laboratories for Radiologic Image Research, Department of Radiology, University of Chicago, Chicago, IL","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112418791","display_name":"Charles E. Metz","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Charles E. Metz","raw_affiliation_strings":["1From the Kurt Rossmann Laboratories for Radiologic Image Research, Department of Radiology, University of Chicago, Chicago, IL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"1From the Kurt Rossmann Laboratories for Radiologic Image Research, Department of Radiology, University of Chicago, Chicago, IL","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111726867","display_name":"Kunio Doi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kunio Doi","raw_affiliation_strings":["1From the Kurt Rossmann Laboratories for Radiologic Image Research, Department of Radiology, University of Chicago, Chicago, IL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"1From the Kurt Rossmann Laboratories for Radiologic Image Research, Department of Radiology, University of Chicago, Chicago, IL","institution_ids":["https://openalex.org/I40347166"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I40347166"],"apc_list":null,"apc_paid":null,"fwci":4.3099,"has_fulltext":false,"cited_by_count":60,"citation_normalized_percentile":{"value":0.94244889,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":99},"biblio":{"volume":"253","issue":"3","first_page":"822","last_page":"830"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11894","display_name":"Radiology practices and education","score":0.926800012588501,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11894","display_name":"Radiology practices and education","score":0.926800012588501,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.0210999995470047,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14374","display_name":"Statistical Methods in Epidemiology","score":0.016100000590085983,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/receiver-operating-characteristic","display_name":"receiver operating characteristic","score":0.9760000109672546},{"id":"https://openalex.org/keywords/diagnostic-accuracy","display_name":"diagnostic accuracy","score":0.5950000286102295},{"id":"https://openalex.org/keywords/experimental-design","display_name":"experimental design","score":0.7419999837875366},{"id":"https://openalex.org/keywords/observer-study","display_name":"observer study","score":0.6359999775886536},{"id":"https://openalex.org/keywords/computer-aided-diagnosis","display_name":"computer-aided diagnosis","score":0.8730000257492065},{"id":"https://openalex.org/keywords/computed-tomography","display_name":"computed tomography","score":0.7699999809265137},{"id":"https://openalex.org/keywords/magnetic-resonance-imaging","display_name":"magnetic resonance imaging","score":1.0},{"id":"https://openalex.org/keywords/diagnostic-performance-evaluation","display_name":"diagnostic performance evaluation","score":0.5410000085830688}],"concepts":[{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.8755795359611511},{"id":"https://openalex.org/C58471807","wikidata":"https://www.wikidata.org/wiki/Q327120","display_name":"Receiver operating characteristic","level":2,"score":0.8099856376647949},{"id":"https://openalex.org/C126838900","wikidata":"https://www.wikidata.org/wiki/Q77604","display_name":"Radiology","level":1,"score":0.6119752526283264},{"id":"https://openalex.org/C19527891","wikidata":"https://www.wikidata.org/wiki/Q1120908","display_name":"Medical physics","level":1,"score":0.5418577194213867},{"id":"https://openalex.org/C143409427","wikidata":"https://www.wikidata.org/wiki/Q161238","display_name":"Magnetic resonance imaging","level":2,"score":0.4927206337451935},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.48995938897132874},{"id":"https://openalex.org/C31601959","wikidata":"https://www.wikidata.org/wiki/Q931309","display_name":"Medical imaging","level":2,"score":0.46421530842781067},{"id":"https://openalex.org/C2989005","wikidata":"https://www.wikidata.org/wiki/Q214963","display_name":"Nuclear medicine","level":1,"score":0.33757877349853516},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21670633554458618},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.08375179767608643},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D003952","descriptor_name":"Diagnostic Imaging","qualifier_ui":"Q000706","qualifier_name":"statistics & numerical data","is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D010506","descriptor_name":"Periodicals as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D011871","descriptor_name":"Radiology","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012372","descriptor_name":"ROC Curve","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012984","descriptor_name":"Software","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D035843","descriptor_name":"Biomedical Research","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":4,"locations":[{"id":"doi:10.1148/radiol.2533081632","doi":"https://doi.org/10.1148/radiol.2533081632","is_oa":false,"landing_page_url":"https://doi.org/10.1148/radiol.2533081632","pdf_url":null,"source":{"id":"https://openalex.org/S50280174","display_name":"Radiology","issn_l":"0033-8419","issn":["0033-8419","1527-1315"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","dongbi-a","doyens","jufo-3","ki-jl-3","medline","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315931","host_organization_name":"Radiological Society of North America","host_organization_lineage":["https://openalex.org/P4310315931"],"host_organization_lineage_names":["Radiological Society of North America"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Radiology","raw_type":"journal-article"},{"id":"pmh:oai:europepmc.org:2028369","doi":null,"is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/2786192","pdf_url":null,"source":{"id":"https://openalex.org/S4306400806","display_name":"Europe PMC (PubMed Central)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1303153112","host_organization_name":"European Bioinformatics Institute","host_organization_lineage":["https://openalex.org/I1303153112"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Text"},{"id":"pmh:oai:pubmedcentral.nih.gov:2786192","doi":null,"is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/2786192","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Radiology","raw_type":"Text"},{"id":"pmid:19864510","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/19864510","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Radiology","raw_type":"Journal Article"}],"best_oa_location":{"id":"pmh:oai:europepmc.org:2028369","doi":null,"is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/2786192","pdf_url":null,"source":{"id":"https://openalex.org/S4306400806","display_name":"Europe PMC (PubMed Central)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1303153112","host_organization_name":"European Bioinformatics Institute","host_organization_lineage":["https://openalex.org/I1303153112"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Text"},"sustainable_development_goals":[{"display_name":"Good health and well-being","id":"https://metadata.un.org/sdg/3","score":0.8048999905586243}],"study_designs":[{"display_name":"Systematic Review","id":"https://openalex.org/study-designs/systematic-review"}],"awards":[{"id":"https://openalex.org/G2302438475","display_name":"Improved DBM ROC Methods for Diagnostic Radiology","funder_award_id":"R01EB000863","funder_id":"https://openalex.org/F4320337363","funder_display_name":"National Institute of Biomedical Imaging and Bioengineering"},{"id":"https://openalex.org/G4737476093","display_name":"CAD for CT Nodules in Lung Cancer Detection","funder_award_id":"R01CA098119","funder_id":"https://openalex.org/F4320337351","funder_display_name":"National Cancer Institute"}],"funders":[{"id":"https://openalex.org/F4320337351","display_name":"National Cancer Institute","ror":"https://ror.org/040gcmg81"},{"id":"https://openalex.org/F4320337363","display_name":"National Institute of Biomedical Imaging and Bioengineering","ror":"https://ror.org/00372qc85"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W1509445416","https://openalex.org/W1539462193","https://openalex.org/W1555798572","https://openalex.org/W1604979160","https://openalex.org/W1784695092","https://openalex.org/W1968114652","https://openalex.org/W1972760446","https://openalex.org/W1972886619","https://openalex.org/W1979828965","https://openalex.org/W1990748933","https://openalex.org/W1992489994","https://openalex.org/W1996064393","https://openalex.org/W2000056517","https://openalex.org/W2035950812","https://openalex.org/W2048352318","https://openalex.org/W2054953567","https://openalex.org/W2074884761","https://openalex.org/W2089114844","https://openalex.org/W2092476803","https://openalex.org/W2097841139","https://openalex.org/W2102150307","https://openalex.org/W2103314326","https://openalex.org/W2124537635","https://openalex.org/W2137194896","https://openalex.org/W2149539590","https://openalex.org/W2154212573","https://openalex.org/W2157825442","https://openalex.org/W2339926369","https://openalex.org/W2921629193","https://openalex.org/W4232875550","https://openalex.org/W4234328703","https://openalex.org/W4293007919","https://openalex.org/W4388317518"],"related_works":["https://openalex.org/W2092476803","https://openalex.org/W2007608640","https://openalex.org/W2100128827","https://openalex.org/W2068479406","https://openalex.org/W2037711351","https://openalex.org/W2033090952","https://openalex.org/W2067972749","https://openalex.org/W2060136512","https://openalex.org/W2035950812","https://openalex.org/W2061393790"],"abstract_inverted_index":{"PURPOSE:":[0],"To":[1],"provide":[2],"a":[3,51,130],"broad":[4],"perspective":[5],"concerning":[6],"the":[7,69,77,91,102,166,217],"recent":[8],"use":[9],"of":[10,68,90,101,124,129,183,185],"receiver":[11],"operating":[12,72],"characteristic":[13],"(ROC)":[14],"analysis":[15,201],"in":[16,24,60,192,205,212,221],"medical":[17,35],"imaging":[18,33,83,168,171],"by":[19,49,76],"reviewing":[20],"ROC":[21,38,186,200,218],"studies":[22,46,93,104,151,187,219],"published":[23,191],"Radiology":[25,61,193,222],"between":[26,62],"1997":[27,63],"and":[28,37,64,66,158,173,230],"2006":[29,65],"for":[30],"experimental":[31],"design,":[32],"modality,":[34],"condition,":[36],"paradigm.":[39],"MATERIALS":[40],"AND":[41],"METHODS:":[42],"Two":[43,97],"hundred":[44,98],"ninety-five":[45],"were":[47,80,87,94,152,159,176,223],"obtained":[48,137],"conducting":[50],"literature":[52],"search":[53],"with":[54,56,122,138,188],"PubMed":[55],"two":[57],"criteria:":[58],"publication":[59],"occurrence":[67],"phrase":[70],"\"receiver":[71],"characteristic.\"":[73],"Studies":[74],"returned":[75],"query":[78],"that":[79],"not":[81,118,224],"diagnostic":[82,110,214],"procedure":[84],"performance":[85],"evaluations":[86],"excluded.":[88],"Characteristics":[89],"remaining":[92,148],"tabulated.":[95],"RESULTS:":[96],"thirty-three":[99],"(79.0%)":[100],"295":[103],"reported":[105,220],"findings":[106],"based":[107],"on":[108],"observers'":[109],"judgments":[111],"or":[112,134,142,156,196],"objective":[113],"measurements.":[114],"Forty-three":[115],"(14.6%)":[116],"did":[117],"include":[119],"human":[120,189],"observers,":[121],"most":[123,178],"these":[125],"reporting":[126],"an":[127],"evaluation":[128],"computer-aided":[131],"diagnosis":[132],"system":[133],"functional":[135],"data":[136],"computed":[139],"tomography":[140],"(CT)":[141],"magnetic":[143],"resonance":[144],"(MR)":[145],"imaging.":[146],"The":[147],"19":[149],"(6.4%)":[150],"classified":[153],"as":[154],"reviews":[155],"meta-analyses":[157],"excluded":[160],"from":[161],"our":[162],"subsequent":[163],"analysis.":[164],"Among":[165],"various":[167],"modalities,":[169],"MR":[170],"(46.0%)":[172],"CT":[174],"(25.7%)":[175],"investigated":[177],"frequently.":[179],"Approximately":[180],"60%":[181],"(144":[182],"233)":[184],"observers":[190],"included":[194],"three":[195],"fewer":[197],"observers.":[198],"CONCLUSION:":[199],"is":[202],"widely":[203],"used":[204],"radiologic":[206],"research,":[207],"confirming":[208],"its":[209],"fundamental":[210],"role":[211],"assessing":[213],"performance.":[215],"However,":[216],"always":[225],"adequate":[226],"to":[227],"support":[228],"clear":[229],"clinically":[231],"relevant":[232],"conclusions.":[233]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":4},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":3},{"year":2013,"cited_by_count":6},{"year":2012,"cited_by_count":5}],"updated_date":"2026-11-06T12:05:46.655746","created_date":"2016-06-24T00:00:00"}