{"id":"https://openalex.org/W4406354025","doi":"https://doi.org/10.1109/tetci.2024.3502465","title":"Benchmarking Medical LLMs on Anesthesiology: A Comprehensive Dataset in Chinese","display_name":"Benchmarking Medical LLMs on Anesthesiology: A Comprehensive Dataset in Chinese","publication_year":2025,"publication_date":"2025-01-14","ids":{"openalex":"https://openalex.org/W4406354025","doi":"https://doi.org/10.1109/tetci.2024.3502465"},"language":"en","primary_location":{"id":"doi:10.1109/tetci.2024.3502465","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tetci.2024.3502465","pdf_url":null,"source":{"id":"https://openalex.org/S4210210251","display_name":"IEEE Transactions on Emerging Topics in Computational Intelligence","issn_l":"2471-285X","issn":["2471-285X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Emerging Topics in Computational Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101712441","display_name":"Bohao Zhou","orcid":"https://orcid.org/0009-0001-9010-895X"},"institutions":[{"id":"https://openalex.org/I4210162190","display_name":"China University of Petroleum, East China","ror":"https://ror.org/05gbn2817","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210162190"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bohao Zhou","raw_affiliation_strings":["College of Control Science and Engineering, China University of Petroleum East China, Qingdao, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Control Science and Engineering, China University of Petroleum East China, Qingdao, China","institution_ids":["https://openalex.org/I4210162190"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074672983","display_name":"Yibing Zhan","orcid":"https://orcid.org/0000-0003-3180-0484"},"institutions":[{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yibing Zhan","raw_affiliation_strings":["JD Explore Academy, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3180-0484","affiliations":[{"raw_affiliation_string":"JD Explore Academy, Beijing, China","institution_ids":["https://openalex.org/I4210103986"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101393653","display_name":"Zhonghai Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210162190","display_name":"China University of Petroleum, East China","ror":"https://ror.org/05gbn2817","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210162190"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhonghai Wang","raw_affiliation_strings":["College of Control Science and Engineering, China University of Petroleum East China, Qingdao, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Control Science and Engineering, China University of Petroleum East China, Qingdao, China","institution_ids":["https://openalex.org/I4210162190"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103083909","display_name":"Yanhong Li","orcid":"https://orcid.org/0000-0002-5449-729X"},"institutions":[{"id":"https://openalex.org/I189210763","display_name":"Yunnan University","ror":"https://ror.org/0040axw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I189210763"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanhong Li","raw_affiliation_strings":["School of Information Science and Engineering, Yunnan University, Kunming, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Information Science and Engineering, Yunnan University, Kunming, China","institution_ids":["https://openalex.org/I189210763"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100393892","display_name":"Chong Zhang","orcid":"https://orcid.org/0000-0001-8913-8206"},"institutions":[{"id":"https://openalex.org/I189210763","display_name":"Yunnan University","ror":"https://ror.org/0040axw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I189210763"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chong Zhang","raw_affiliation_strings":["School of Information Science and Engineering, Yunnan University, Kunming, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Information Science and Engineering, Yunnan University, Kunming, China","institution_ids":["https://openalex.org/I189210763"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085309099","display_name":"Baosheng Yu","orcid":"https://orcid.org/0000-0002-0761-7893"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Baosheng Yu","raw_affiliation_strings":["The University of Sydney, Darlington, NSW, Australia"],"raw_orcid":"https://orcid.org/0000-0002-0761-7893","affiliations":[{"raw_affiliation_string":"The University of Sydney, Darlington, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100456723","display_name":"Liang Ding","orcid":"https://orcid.org/0000-0001-8976-2084"},"institutions":[{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liang Ding","raw_affiliation_strings":["JD Explore Academy, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8976-2084","affiliations":[{"raw_affiliation_string":"JD Explore Academy, Beijing, China","institution_ids":["https://openalex.org/I4210103986"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101327489","display_name":"Hua Jin","orcid":null},"institutions":[{"id":"https://openalex.org/I4210091044","display_name":"First People's Hospital of Yunnan Province","ror":"https://ror.org/00c099g34","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210091044"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hua Jin","raw_affiliation_strings":["First People's Hospital of Yunnan Province, Kunming, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"First People's Hospital of Yunnan Province, Kunming, China","institution_ids":["https://openalex.org/I4210091044"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100444156","display_name":"Weifeng Liu","orcid":"https://orcid.org/0000-0002-5388-9080"},"institutions":[{"id":"https://openalex.org/I4210162190","display_name":"China University of Petroleum, East China","ror":"https://ror.org/05gbn2817","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210162190"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weifeng Liu","raw_affiliation_strings":["College of Control Science and Engineering, China University of Petroleum East China, Qingdao, China"],"raw_orcid":"https://orcid.org/0000-0002-5388-9080","affiliations":[{"raw_affiliation_string":"College of Control Science and Engineering, China University of Petroleum East China, Qingdao, China","institution_ids":["https://openalex.org/I4210162190"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039205620","display_name":"Xiongbin Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I26080491","display_name":"Kunming Medical University","ror":"https://ror.org/038c3w259","country_code":"CN","type":"education","lineage":["https://openalex.org/I26080491"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiongbin Wang","raw_affiliation_strings":["Second Affiliated Hospital of Kunming Medical University, Kunming, China"],"raw_orcid":"https://orcid.org/0009-0003-0045-8724","affiliations":[{"raw_affiliation_string":"Second Affiliated Hospital of Kunming Medical University, Kunming, China","institution_ids":["https://openalex.org/I26080491"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5067600725","display_name":"Dapeng Tao","orcid":"https://orcid.org/0000-0003-0783-5273"},"institutions":[{"id":"https://openalex.org/I189210763","display_name":"Yunnan University","ror":"https://ror.org/0040axw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I189210763"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dapeng Tao","raw_affiliation_strings":["School of Information Science and Engineering, Yunnan University, Kunming, China"],"raw_orcid":"https://orcid.org/0000-0003-0783-5273","affiliations":[{"raw_affiliation_string":"School of Information Science and Engineering, Yunnan University, Kunming, China","institution_ids":["https://openalex.org/I189210763"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.5198,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.97994131,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"9","issue":"4","first_page":"3057","last_page":"3071"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12168","display_name":"Health and Medical Research Impacts","score":0.8149999976158142,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T12168","display_name":"Health and Medical Research Impacts","score":0.8149999976158142,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.7544999718666077,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T14134","display_name":"Medical and Biological Sciences","score":0.7186999917030334,"subfield":{"id":"https://openalex.org/subfields/2702","display_name":"Anatomy"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.9120688438415527},{"id":"https://openalex.org/keywords/anesthesiology","display_name":"Anesthesiology","score":0.6899296045303345},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.5001468658447266},{"id":"https://openalex.org/keywords/intensive-care-medicine","display_name":"Intensive care medicine","score":0.34530338644981384},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.2389264702796936},{"id":"https://openalex.org/keywords/pathology","display_name":"Pathology","score":0.14196354150772095}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.9120688438415527},{"id":"https://openalex.org/C2779526319","wikidata":"https://www.wikidata.org/wiki/Q615057","display_name":"Anesthesiology","level":2,"score":0.6899296045303345},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.5001468658447266},{"id":"https://openalex.org/C177713679","wikidata":"https://www.wikidata.org/wiki/Q679690","display_name":"Intensive care medicine","level":1,"score":0.34530338644981384},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.2389264702796936},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.14196354150772095},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tetci.2024.3502465","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tetci.2024.3502465","pdf_url":null,"source":{"id":"https://openalex.org/S4210210251","display_name":"IEEE Transactions on Emerging Topics in Computational Intelligence","issn_l":"2471-285X","issn":["2471-285X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Emerging Topics in Computational Intelligence","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2123672980","https://openalex.org/W2270593706","https://openalex.org/W2896457183","https://openalex.org/W2923014074","https://openalex.org/W3112856102","https://openalex.org/W3170998314","https://openalex.org/W4283165972","https://openalex.org/W4285294723","https://openalex.org/W4286233477","https://openalex.org/W4312341782","https://openalex.org/W4312505683","https://openalex.org/W4327946446","https://openalex.org/W4366420437","https://openalex.org/W4377030497","https://openalex.org/W4378189609","https://openalex.org/W4384071683","https://openalex.org/W4385571124","https://openalex.org/W4385834497","https://openalex.org/W4387847108","https://openalex.org/W4388033920","https://openalex.org/W4389520222","https://openalex.org/W4389520259","https://openalex.org/W4390116719","https://openalex.org/W4392391178","https://openalex.org/W4392773006","https://openalex.org/W4392901645","https://openalex.org/W4392939486","https://openalex.org/W4393156546","https://openalex.org/W4394892516","https://openalex.org/W4401043132","https://openalex.org/W4402670905","https://openalex.org/W6636510571","https://openalex.org/W6680532216","https://openalex.org/W6682631176","https://openalex.org/W6761205521","https://openalex.org/W6762392948","https://openalex.org/W6778883912","https://openalex.org/W6847020704","https://openalex.org/W6850242479","https://openalex.org/W6850724024","https://openalex.org/W6852178287","https://openalex.org/W6852944568","https://openalex.org/W6853244322","https://openalex.org/W6856398428","https://openalex.org/W6856745014","https://openalex.org/W6856800273","https://openalex.org/W6857797255","https://openalex.org/W6861679459","https://openalex.org/W6862207580"],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W3031052312","https://openalex.org/W4238897586","https://openalex.org/W4389568370","https://openalex.org/W3032375762","https://openalex.org/W435179959","https://openalex.org/W1995515455","https://openalex.org/W2080531066","https://openalex.org/W2619091065"],"abstract_inverted_index":{"With":[0,124],"the":[1,21,27,57,65,81,125,163,177,184,190],"recent":[2],"success":[3],"of":[4,23,29,67,134,165,173,179,192],"large":[5],"language":[6],"models":[7],"(LLMs),":[8],"interest":[9],"in":[10,35,53,162,176],"developing":[11],"them":[12],"for":[13,70,94,109,121,168,195],"medical":[14,30,68,118,136,166,174,193],"domains":[15],"has":[16],"increased.":[17],"However,":[18],"due":[19],"to":[20],"lack":[22],"benchmark":[24,51,63],"datasets,":[25],"evaluating":[26],"capabilities":[28],"LLMs":[31,69,167,194],"remains":[32],"challenging,":[33],"particularly":[34],"highly":[36],"specialized":[37],"fields":[38],"such":[39,138,147],"as":[40,56,139,148],"anesthesiology.":[41,196],"To":[42],"address":[43],"this":[44],"gap,":[45],"we":[46,129],"introduce":[47],"a":[48,131],"comprehensive":[49],"anesthesiology":[50,71,169],"dataset":[52,187],"Chinese,":[54],"known":[55],"Chinese":[58],"Anesthesiology":[59],"Benchmark":[60],"(CAB).":[61],"This":[62],"facilitates":[64],"evaluation":[66,133,153],"across":[72],"three":[73],"crucial":[74],"dimensions:":[75],"knowledge,":[76],"application,":[77],"and":[78,92,107,112,141,143,150],"safety.":[79],"Specifically,":[80],"CAB":[82,127,186],"provides":[83],"more":[84,97],"than":[85,98],"8":[86],"k":[87,100],"questions":[88,101],"collected":[89,102],"from":[90,103,115],"examinations":[91],"books":[93],"knowledge-level":[95],"evaluation;":[96,111],"2":[99],"online":[104],"anesthesia":[105,117],"consultations":[106],"hospitals":[108],"application-level":[110],"136":[113],"tests":[114],"seven":[116],"care":[119],"scenarios":[120],"safety-level":[122],"evaluation.":[123],"proposed":[126,185],"dataset,":[128],"conducted":[130],"thorough":[132],"six":[135],"LLMs,":[137,146],"Bianque-2":[140],"HuatuoGPT-13B,":[142],"eleven":[144],"general":[145],"Qwen-7B-Chat":[149],"GPT-4.":[151],"The":[152],"results":[154],"revealed":[155],"that":[156,183],"there":[157],"are":[158],"still":[159],"clear":[160],"gaps":[161],"capacities":[164],"compared":[170],"with":[171],"those":[172],"students":[175],"field":[178],"anesthesia.":[180],"We":[181],"hope":[182],"can":[188],"facilitate":[189],"development":[191]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":5}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}