{"id":"https://openalex.org/W4404963495","doi":"https://doi.org/10.2196/64284","title":"Performance Evaluation and Implications of Large Language Models in Radiology Board Exams: Prospective Comparative Analysis","display_name":"Performance Evaluation and Implications of Large Language Models in Radiology Board Exams: Prospective Comparative Analysis","alternate_titles":[],"publication_year":2024,"publication_date":"2024-12-03","ids":{"openalex":"https://openalex.org/W4404963495","doi":"https://doi.org/10.2196/64284","pmid":"https://pubmed.ncbi.nlm.nih.gov/39819381","pmcid":"https://www.ncbi.nlm.nih.gov/pmc/articles/11756834"},"language":"en","primary_location":{"id":"doi:10.2196/64284","doi":"https://doi.org/10.2196/64284","is_oa":true,"landing_page_url":"https://doi.org/10.2196/64284","pdf_url":"https://mededu.jmir.org/2025/1/e64284/PDF","source":{"id":"https://openalex.org/S4210212741","display_name":"JMIR Medical Education","issn_l":"2369-3762","issn":["2369-3762"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","doyens","jufo-1","ki-jl-1","medline","norway-1","oarl","poland-20","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320608","host_organization_name":"JMIR Publications","host_organization_lineage":["https://openalex.org/P4310320608"],"host_organization_lineage_names":["JMIR Publications"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"JMIR Medical Education","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W4404963495"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://mededu.jmir.org/2025/1/e64284/PDF","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036285374","display_name":"Boxiong Wei","orcid":"https://orcid.org/0000-0001-6206-9319","observed_orcids":["https://orcid.org/0000-0001-6206-9319"]},"institutions":[{"id":"https://openalex.org/I4210130930","display_name":"Peking University First Hospital","ror":"https://ror.org/02z1vqm45","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210130930"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Boxiong Wei","raw_affiliation_strings":["Department of Ultrasound, Peking University First Hospital, 8 Xishiku Rd, Xicheng District, Beijing, 100034, China, 86 13132150190, 86 314521"],"raw_orcid":"https://orcid.org/0000-0001-6206-9319","affiliations":[{"raw_affiliation_string":"Department of Ultrasound, Peking University First Hospital, 8 Xishiku Rd, Xicheng District, Beijing, 100034, China, 86 13132150190, 86 314521","institution_ids":["https://openalex.org/I4210130930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5036285374"],"corresponding_institution_ids":["https://openalex.org/I4210130930"],"apc_list":{"value":1873,"currency":"USD","value_usd":1873},"apc_paid":{"value":1873,"currency":"USD","value_usd":1873},"fwci":2.0073,"has_fulltext":true,"cited_by_count":28,"citation_normalized_percentile":{"value":0.86342984,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"11","issue":null,"first_page":"e64284","last_page":"e64284"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":1.0,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":1.0,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11894","display_name":"Radiology practices and education","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.0,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":1.0},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.9729999899864197},{"id":"https://openalex.org/keywords/claude","display_name":"Claude","score":0.9259999990463257},{"id":"https://openalex.org/keywords/google-bard","display_name":"Google Bard","score":0.8100000023841858},{"id":"https://openalex.org/keywords/gemini-pro","display_name":"Gemini Pro","score":0.984000027179718},{"id":"https://openalex.org/keywords/multiple-choice-question-accuracy","display_name":"multiple-choice question accuracy","score":0.8169999718666077},{"id":"https://openalex.org/keywords/cognitive-levels","display_name":"cognitive levels","score":0.47999998927116394},{"id":"https://openalex.org/keywords/physician-specialty-differences","display_name":"physician specialty differences","score":0.5139999985694885},{"id":"https://openalex.org/keywords/radiology-education","display_name":"radiology education","score":0.7670000195503235},{"id":"https://openalex.org/keywords/domain-specific-training-data","display_name":"domain-specific training data","score":0.6430000066757202}],"concepts":[{"id":"https://openalex.org/C43169469","wikidata":"https://www.wikidata.org/wiki/Q580922","display_name":"Preprint","level":2,"score":0.9363701343536377},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.37834692001342773},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.09432509541511536}],"mesh":[{"descriptor_ui":"D000098342","descriptor_name":"Large Language Models","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D002983","descriptor_name":"Clinical Competence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D004521","descriptor_name":"Educational Measurement","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D011446","descriptor_name":"Prospective Studies","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D011871","descriptor_name":"Radiology","qualifier_ui":"Q000193","qualifier_name":"education","is_major_topic":true}],"locations_count":5,"locations":[{"id":"doi:10.2196/64284","doi":"https://doi.org/10.2196/64284","is_oa":true,"landing_page_url":"https://doi.org/10.2196/64284","pdf_url":"https://mededu.jmir.org/2025/1/e64284/PDF","source":{"id":"https://openalex.org/S4210212741","display_name":"JMIR Medical Education","issn_l":"2369-3762","issn":["2369-3762"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","doyens","jufo-1","ki-jl-1","medline","norway-1","oarl","poland-20","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320608","host_organization_name":"JMIR Publications","host_organization_lineage":["https://openalex.org/P4310320608"],"host_organization_lineage_names":["JMIR Publications"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"JMIR Medical Education","raw_type":"journal-article"},{"id":"doi:10.2196/preprints.64284","doi":"https://doi.org/10.2196/preprints.64284","is_oa":true,"landing_page_url":"http://dx.doi.org/10.2196/preprints.64284","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},{"id":"pmh:oai:pubmedcentral.nih.gov:11756834","doi":null,"is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/11756834","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"JMIR Med Educ","raw_type":"Text"},{"id":"pmid:39819381","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/39819381","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"JMIR medical education","raw_type":"Journal Article"},{"id":"pmh:oai:doaj.org/article:e09f6dc558884493898f9c22a1983de9","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/e09f6dc558884493898f9c22a1983de9","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"JMIR Medical Education, Vol 11, Pp e64284-e64284 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.2196/64284","doi":"https://doi.org/10.2196/64284","is_oa":true,"landing_page_url":"https://doi.org/10.2196/64284","pdf_url":"https://mededu.jmir.org/2025/1/e64284/PDF","source":{"id":"https://openalex.org/S4210212741","display_name":"JMIR Medical Education","issn_l":"2369-3762","issn":["2369-3762"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","doyens","jufo-1","ki-jl-1","medline","norway-1","oarl","poland-20","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320608","host_organization_name":"JMIR Publications","host_organization_lineage":["https://openalex.org/P4310320608"],"host_organization_lineage_names":["JMIR Publications"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"JMIR Medical Education","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being","score":0.5683000087738037},{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.4156000018119812}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404963495.pdf","grobid_xml":"https://content.openalex.org/works/W4404963495.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W2051885079","https://openalex.org/W2162821268","https://openalex.org/W2783687327","https://openalex.org/W2785645041","https://openalex.org/W2794518994","https://openalex.org/W2890106926","https://openalex.org/W2963353367","https://openalex.org/W2976398475","https://openalex.org/W3011149445","https://openalex.org/W3012043879","https://openalex.org/W3013578857","https://openalex.org/W3013681994","https://openalex.org/W3014301887","https://openalex.org/W3138769367","https://openalex.org/W3138961261","https://openalex.org/W3150212014","https://openalex.org/W3177044444","https://openalex.org/W4229036772","https://openalex.org/W4376640706","https://openalex.org/W4376640806","https://openalex.org/W4387327030","https://openalex.org/W4390393653","https://openalex.org/W4392665596","https://openalex.org/W4394877296"],"related_works":["https://openalex.org/W4400840172","https://openalex.org/W7126028701","https://openalex.org/W7166061906","https://openalex.org/W4405010956","https://openalex.org/W4404029770","https://openalex.org/W4403998249","https://openalex.org/W7154696000","https://openalex.org/W4416808600","https://openalex.org/W4400642441"],"abstract_inverted_index":{"Background:":[0],"Artificial":[1],"intelligence":[2],"advancements":[3],"have":[4],"enabled":[5],"large":[6,25,217],"language":[7,26,218],"models":[8],"to":[9,121,167,215],"significantly":[10,88],"impact":[11],"radiology":[12,38,50],"education":[13,203],"and":[14,34,66,72,78,111,136,156,174,182,187,196,204],"diagnostic":[15,192],"accuracy.":[16],"Objective:":[17],"This":[18],"study":[19,207],"evaluates":[20],"the":[21,83,209],"performance":[22],"of":[23,45,98,152,161],"mainstream":[24],"models,":[27],"including":[28],"GPT-4,":[29],"Claude,":[30,129],"Bard,":[31,135],"Tongyi":[32,106,144,197],"Qianwen,":[33],"Gemini":[35,112,142],"Pro,":[36],"in":[37,179,201,221],"board":[39,51],"exams.":[40],"Methods:":[41],"A":[42],"comparative":[43],"analysis":[44],"150":[46],"multiple-choice":[47],"questions":[48,65],"from":[49],"exams":[52],"without":[53],"images":[54],"was":[55],"conducted.":[56],"Models":[57],"were":[58,67,123],"assessed":[59],"on":[60],"their":[61],"accuracy":[62,85,97,151],"for":[63,128,134,141,211],"text-based":[64],"categorized":[68],"by":[69],"cognitive":[70],"levels":[71],"medical":[73,202],"specialties":[74],"using":[75],"\u03c72":[76],"tests":[77],"ANOVA.":[79],"Results:":[80],"GPT-4":[81,122,177,195],"achieved":[82,95],"highest":[84],"(83.3%,":[86],"125/150),":[87],"outperforming":[89],"all":[90],"other":[91],"models.":[92],"Specifically,":[93],"Claude":[94,186],"an":[96,150,158],"62%":[99],"(93/150;":[100],"P<.001),":[101,105],"Bard":[102,188],"54.7%":[103],"(82/150;":[104],"Qianwen":[107,145,198],"70.7%":[108,153],"(106/150;":[109,154],"P=.009),":[110],"Pro":[113],"55.3%":[114],"(83/150;":[115],"P<.001).":[116],"The":[117,206],"odds":[118,159],"ratios":[119],"compared":[120,166],"0.33":[124],"(95%":[125,131,138,163],"CI":[126,132,139,164],"0.18-0.60)":[127],"0.24":[130],"0.13-0.44)":[133],"0.25":[137],"0.14-0.45)":[140],"Pro.":[143],"performed":[146],"relatively":[147],"well":[148],"with":[149,176,190],"P=0.02)":[155],"had":[157],"ratio":[160],"0.48":[162],"0.27-0.87)":[165],"GPT-4.":[168],"Performance":[169],"varied":[170],"across":[171],"question":[172],"types":[173],"specialties,":[175],"excelling":[178],"both":[180],"lower-order":[181],"higher-order":[183],"questions,":[184],"while":[185],"struggled":[189],"complex":[191],"questions.":[193],"Conclusions:":[194],"show":[199],"promise":[200],"training.":[205],"emphasizes":[208],"need":[210],"domain-specific":[212],"training":[213],"datasets":[214],"enhance":[216],"models'":[219],"effectiveness":[220],"specialized":[222],"fields":[223],"like":[224],"radiology.":[225]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4404963495","counts_by_year":[{"year":2026,"cited_by_count":14},{"year":2025,"cited_by_count":14}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2024-12-04T00:00:00"}
