{"id":"https://openalex.org/W4372047097","doi":"https://doi.org/10.4174/astr.2023.104.5.269","title":"ChatGPT goes to the operating room: evaluating GPT-4 performance and its potential in surgical education and training in the era of large language models","display_name":"ChatGPT goes to the operating room: evaluating GPT-4 performance and its potential in surgical education and training in the era of large language models","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4372047097","doi":"https://doi.org/10.4174/astr.2023.104.5.269","pmid":"https://pubmed.ncbi.nlm.nih.gov/37179699","pmcid":"https://www.ncbi.nlm.nih.gov/pmc/articles/10172028"},"language":"en","primary_location":{"id":"doi:10.4174/astr.2023.104.5.269","doi":"https://doi.org/10.4174/astr.2023.104.5.269","is_oa":true,"landing_page_url":"https://doi.org/10.4174/astr.2023.104.5.269","pdf_url":"http://astr.or.kr/Synapse/Data/PDFData/6037ASTR/astr-104-269.pdf","source":{"id":"https://openalex.org/S4210169889","display_name":"Annals of Surgical Treatment and Research","issn_l":"2288-6575","issn":["2288-6575","2288-6796"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-d","jufo-1","kci-registered","poland-70","russia-white-list-2"],"host_organization":"https://openalex.org/P4322632799","host_organization_name":"Korean Surgical Society","host_organization_lineage":["https://openalex.org/P4322632799"],"host_organization_lineage_names":["Korean Surgical Society"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Annals of Surgical Treatment and Research","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"http://astr.or.kr/Synapse/Data/PDFData/6037ASTR/astr-104-269.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043814482","display_name":"Namkee Oh","orcid":"https://orcid.org/0000-0002-6594-8973","observed_orcids":["https://orcid.org/0000-0002-6594-8973"]},"institutions":[{"id":"https://openalex.org/I2802194831","display_name":"Samsung Medical Center","ror":"https://ror.org/05a15z872","country_code":"KR","type":"healthcare","lineage":["https://openalex.org/I2250650973","https://openalex.org/I2802194831"]},{"id":"https://openalex.org/I848706","display_name":"Sungkyunkwan University","ror":"https://ror.org/04q78tk20","country_code":"KR","type":"education","lineage":["https://openalex.org/I848706"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Namkee Oh","raw_affiliation_strings":["Department of Surgery, Samsung Medical Center, Sungkyunkwan University School of Medicine, Seoul, Korea"],"raw_orcid":"https://orcid.org/0000-0002-6594-8973","affiliations":[{"raw_affiliation_string":"Department of Surgery, Samsung Medical Center, Sungkyunkwan University School of Medicine, Seoul, Korea","institution_ids":["https://openalex.org/I2802194831","https://openalex.org/I848706"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060910033","display_name":"Gyu\u2010Seong Choi","orcid":"https://orcid.org/0000-0003-2545-3105","observed_orcids":["https://orcid.org/0000-0003-2545-3105"]},"institutions":[{"id":"https://openalex.org/I2802194831","display_name":"Samsung Medical Center","ror":"https://ror.org/05a15z872","country_code":"KR","type":"healthcare","lineage":["https://openalex.org/I2250650973","https://openalex.org/I2802194831"]},{"id":"https://openalex.org/I848706","display_name":"Sungkyunkwan University","ror":"https://ror.org/04q78tk20","country_code":"KR","type":"education","lineage":["https://openalex.org/I848706"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Gyu-Seong Choi","raw_affiliation_strings":["Department of Surgery, Samsung Medical Center, Sungkyunkwan University School of Medicine, Seoul, Korea"],"raw_orcid":"https://orcid.org/0000-0003-2545-3105","affiliations":[{"raw_affiliation_string":"Department of Surgery, Samsung Medical Center, Sungkyunkwan University School of Medicine, Seoul, Korea","institution_ids":["https://openalex.org/I2802194831","https://openalex.org/I848706"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101676772","display_name":"Woo Yong Lee","orcid":"https://orcid.org/0000-0002-9558-9019","observed_orcids":["https://orcid.org/0000-0002-9558-9019"]},"institutions":[{"id":"https://openalex.org/I2802194831","display_name":"Samsung Medical Center","ror":"https://ror.org/05a15z872","country_code":"KR","type":"healthcare","lineage":["https://openalex.org/I2250650973","https://openalex.org/I2802194831"]},{"id":"https://openalex.org/I848706","display_name":"Sungkyunkwan University","ror":"https://ror.org/04q78tk20","country_code":"KR","type":"education","lineage":["https://openalex.org/I848706"]}],"countries":["KR"],"is_corresponding":true,"raw_author_name":"Woo Yong Lee","raw_affiliation_strings":["Department of Surgery, Samsung Medical Center, Sungkyunkwan University School of Medicine, Seoul, Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Surgery, Samsung Medical Center, Sungkyunkwan University School of Medicine, Seoul, Korea","institution_ids":["https://openalex.org/I2802194831","https://openalex.org/I848706"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5101676772"],"corresponding_institution_ids":["https://openalex.org/I2802194831","https://openalex.org/I848706"],"apc_list":null,"apc_paid":null,"fwci":7.7566,"has_fulltext":true,"cited_by_count":204,"citation_normalized_percentile":{"value":0.98164015,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"104","issue":"5","first_page":"269","last_page":"269"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10916","display_name":"Surgical Simulation and Training","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/2746","display_name":"Surgery"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/chatgpt","display_name":"ChatGPT","score":0.8889999985694885},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.9890000224113464},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8420000076293945},{"id":"https://openalex.org/keywords/surgical-training","display_name":"surgical training","score":0.8889999985694885},{"id":"https://openalex.org/keywords/model-performance-comparison","display_name":"model performance comparison","score":0.5440000295639038},{"id":"https://openalex.org/keywords/gpt-3-5","display_name":"GPT-3.5","score":0.8640000224113464},{"id":"https://openalex.org/keywords/artificial-intelligence-in-healthcare","display_name":"artificial intelligence in healthcare","score":0.6420000195503235}],"concepts":[{"id":"https://openalex.org/C186282968","wikidata":"https://www.wikidata.org/wiki/Q1434261","display_name":"McNemar's test","level":2,"score":0.8797601461410522},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.46863436698913574},{"id":"https://openalex.org/C19527891","wikidata":"https://www.wikidata.org/wiki/Q1120908","display_name":"Medical physics","level":1,"score":0.42770880460739136},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4102700352668762},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.3354550004005432},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.3225787281990051},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.17368286848068237},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12765702605247498},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.4174/astr.2023.104.5.269","doi":"https://doi.org/10.4174/astr.2023.104.5.269","is_oa":true,"landing_page_url":"https://doi.org/10.4174/astr.2023.104.5.269","pdf_url":"http://astr.or.kr/Synapse/Data/PDFData/6037ASTR/astr-104-269.pdf","source":{"id":"https://openalex.org/S4210169889","display_name":"Annals of Surgical Treatment and Research","issn_l":"2288-6575","issn":["2288-6575","2288-6796"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-d","jufo-1","kci-registered","poland-70","russia-white-list-2"],"host_organization":"https://openalex.org/P4322632799","host_organization_name":"Korean Surgical Society","host_organization_lineage":["https://openalex.org/P4322632799"],"host_organization_lineage_names":["Korean Surgical Society"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Annals of Surgical Treatment and Research","raw_type":"journal-article"},{"id":"doi:10.1101/2023.03.16.23287340","doi":"https://doi.org/10.1101/2023.03.16.23287340","is_oa":true,"landing_page_url":"https://doi.org/10.1101/2023.03.16.23287340","pdf_url":"https://www.medrxiv.org/content/medrxiv/early/2023/03/31/2023.03.16.23287340.full.pdf","source":{"id":"https://openalex.org/S3005729997","display_name":"medRxiv","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310315909"],"host_organization_lineage_names":["Cold Spring Harbor Laboratory Press"],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"medRxiv","raw_type":"posted-content"},{"id":"pmh:oai:pubmedcentral.nih.gov:10172028","doi":null,"is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/10172028","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC10172028/pdf/astr-104-269.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Ann Surg Treat Res","raw_type":"Text"},{"id":"pmid:37179699","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37179699","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Annals of surgical treatment and research","raw_type":"Journal Article"}],"best_oa_location":{"id":"doi:10.4174/astr.2023.104.5.269","doi":"https://doi.org/10.4174/astr.2023.104.5.269","is_oa":true,"landing_page_url":"https://doi.org/10.4174/astr.2023.104.5.269","pdf_url":"http://astr.or.kr/Synapse/Data/PDFData/6037ASTR/astr-104-269.pdf","source":{"id":"https://openalex.org/S4210169889","display_name":"Annals of Surgical Treatment and Research","issn_l":"2288-6575","issn":["2288-6575","2288-6796"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-d","jufo-1","kci-registered","poland-70","russia-white-list-2"],"host_organization":"https://openalex.org/P4322632799","host_organization_name":"Korean Surgical Society","host_organization_lineage":["https://openalex.org/P4322632799"],"host_organization_lineage_names":["Korean Surgical Society"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Annals of Surgical Treatment and Research","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/3","score":0.8371999859809875,"display_name":"Good health and well-being"},{"id":"https://metadata.un.org/sdg/4","score":0.6164000034332275,"display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320332195","display_name":"Samsung","ror":"https://ror.org/04w3jy968"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4372047097.pdf","grobid_xml":"https://content.openalex.org/works/W4372047097.grobid-xml"},"referenced_works_count":12,"referenced_works":["https://openalex.org/W1535688809","https://openalex.org/W2088383760","https://openalex.org/W2766894906","https://openalex.org/W2995872180","https://openalex.org/W3007199715","https://openalex.org/W3133702157","https://openalex.org/W4297253404","https://openalex.org/W4313384047","https://openalex.org/W4319662928","https://openalex.org/W4319663047","https://openalex.org/W4320009668","https://openalex.org/W4322718191"],"related_works":["https://openalex.org/W4397293584","https://openalex.org/W4410898220","https://openalex.org/W4409381278","https://openalex.org/W4412744396","https://openalex.org/W4391069701","https://openalex.org/W7119987055","https://openalex.org/W4392900922","https://openalex.org/W4417416364","https://openalex.org/W4385827730","https://openalex.org/W4392847956"],"abstract_inverted_index":{"Purpose:":[0],"This":[1],"study":[2],"aimed":[3],"to":[4,110,120,143],"assess":[5],"the":[6,11,38,91,133,145],"performance":[7,89,100],"of":[8,70,82,130,147],"ChatGPT,":[9,113],"specifically":[10],"GPT-3.5":[12,50,65],"and":[13,22,29,47,51,56,151,162],"GPT-4":[14,52,73,96],"models,":[15],"in":[16,88,157],"understanding":[17],"complex":[18,122],"surgical":[19,27,123],"clinical":[20,124],"information":[21],"its":[23],"potential":[24],"implications":[25],"for":[26],"education":[28],"training.":[30],"Methods:":[31],"The":[32],"dataset":[33],"comprised":[34],"280":[35],"questions":[36],"from":[37,108],"Korean":[39,134],"general":[40,135],"surgery":[41,136],"board":[42,137],"exams":[43],"conducted":[44],"between":[45,90],"2020":[46],"2022.":[48],"Both":[49],"models":[53,92,150],"were":[54,59],"evaluated,":[55],"their":[57],"performances":[58],"compared":[60],"using":[61],"McNemar":[62],"test.":[63],"Results:":[64],"achieved":[66],"an":[67,79,127],"overall":[68,80],"accuracy":[69,81,105,128],"46.8%,":[71],"while":[72],"demonstrated":[74],"a":[75,85,117],"significant":[76],"improvement":[77],"with":[78,104,159],"76.4%,":[83],"indicating":[84],"notable":[86],"difference":[87],"(P":[93],"<":[94],"0.001).":[95],"also":[97],"exhibited":[98],"consistent":[99],"across":[101],"all":[102],"subspecialties,":[103],"rates":[106],"ranging":[107],"63.6%":[109],"83.3%.":[111],"Conclusion:":[112],"particularly":[114],"GPT-4,":[115],"demonstrates":[116],"remarkable":[118],"ability":[119],"understand":[121],"information,":[125],"achieving":[126],"rate":[129],"76.4%":[131],"on":[132],"exam.":[138],"However,":[139],"it":[140],"is":[141],"important":[142],"recognize":[144],"limitations":[146],"large":[148],"language":[149],"ensure":[152],"that":[153],"they":[154],"are":[155],"used":[156],"conjunction":[158],"human":[160],"expertise":[161],"judgment.":[163]},"counts_by_year":[{"year":2026,"cited_by_count":18},{"year":2025,"cited_by_count":65},{"year":2024,"cited_by_count":92},{"year":2023,"cited_by_count":29}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2023-05-07T00:00:00"}