{"id":"https://openalex.org/W4412888515","doi":"https://doi.org/10.18653/v1/2025.findings-acl.518","title":"EMPEC: A Comprehensive Benchmark for Evaluating Large Language Models Across Diverse Healthcare Professions","display_name":"EMPEC: A Comprehensive Benchmark for Evaluating Large Language Models Across Diverse Healthcare Professions","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412888515","doi":"https://doi.org/10.18653/v1/2025.findings-acl.518"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.518","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.518","pdf_url":"https://aclanthology.org/2025.findings-acl.518.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.518.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075367840","display_name":"Zheheng Luo","orcid":"https://orcid.org/0000-0001-8246-5511"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheheng Luo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034535852","display_name":"Chenhan Yuan","orcid":"https://orcid.org/0000-0001-9667-0460"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chenhan Yuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101868563","display_name":"Qianqian Xie","orcid":"https://orcid.org/0000-0002-9588-7454"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qianqian Xie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5077976343","display_name":"Sophia Ananiadou","orcid":"https://orcid.org/0000-0002-4097-9191"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sophia Ananiadou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"9945","last_page":"9958"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.7530999779701233,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.7530999779701233,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8184791803359985},{"id":"https://openalex.org/keywords/health-care","display_name":"Health care","score":0.7027631998062134},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6398830413818359},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.425123393535614},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.33083975315093994},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.0965014100074768},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.08319392800331116}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8184791803359985},{"id":"https://openalex.org/C160735492","wikidata":"https://www.wikidata.org/wiki/Q31207","display_name":"Health care","level":2,"score":0.7027631998062134},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6398830413818359},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.425123393535614},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33083975315093994},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0965014100074768},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.08319392800331116},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.518","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.518","pdf_url":"https://aclanthology.org/2025.findings-acl.518.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.518","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.518","pdf_url":"https://aclanthology.org/2025.findings-acl.518.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.5600000023841858}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412888515.pdf","grobid_xml":"https://content.openalex.org/works/W4412888515.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W4246352526"],"abstract_inverted_index":{"Recent":[0],"advancements":[1,191],"in":[2,10,40,111,117,138,169,188,192,206],"Large":[3,193],"Language":[4,194],"Models":[5,195],"(LLMs)":[6,196],"show":[7],"their":[8,108,132,136],"potential":[9,200],"accurately":[11],"answering":[12],"biomedical":[13],"questions,":[14],"yet":[15],"current":[16],"healthcare":[17,45,57],"benchmarks":[18,164],"primarily":[19],"assess":[20,166],"knowledge":[21,46],"mastered":[22],"by":[23],"medical":[24],"doctors,":[25],"neglecting":[26],"other":[27],"essential":[28],"professions.To":[29],"address":[30],"this":[31],"gap,":[32],"we":[33,101,122,173],"introduce":[34],"the":[35,129,143,153,160,176,199],"Examinations":[36],"for":[37,69,162,181],"Medical":[38],"PErsonnel":[39],"Chinese":[41,150],"(EMPEC),":[42],"a":[43],"comprehensive":[44],"benchmark":[47],"featuring":[48],"157,803":[49],"exam":[50],"questions":[51,126,211],"across":[52],"124":[53],"subjects":[54],"and":[55,64,72,80,98,113,148,151,172,178,185],"20":[56],"professions,":[58],"including":[59,78],"underrepresented":[60],"roles":[61],"like":[62,87],"Optometrists":[63],"Audiologists.Each":[65],"question":[66],"is":[67],"tagged":[68],"release":[70],"time":[71],"source":[73],"authenticity.We":[74],"evaluated":[75],"17":[76],"LLMs,":[77],"proprietary":[79],"open-source":[81],"models,":[82],"finding":[83],"that":[84,103],"while":[85],"models":[86],"GPT-4":[88],"achieved":[89],"over":[90],"75%":[91],"accuracy,":[92],"they":[93],"struggled":[94],"with":[95],"specialised":[96],"fields":[97],"alternative":[99],"medicine.Notably,":[100],"find":[102],"most":[104],"medicalspecific":[105],"LLMs":[106,124],"underperform":[107],"generalpurpose":[109],"counterparts":[110],"EMPEC,":[112],"incorporating":[114],"EMPEC's":[115],"data":[116,184],"fine-tuning":[118],"improves":[119],"performance.In":[120],"addition,":[121],"tested":[123],"on":[125,155],"released":[127],"after":[128],"completion":[130],"of":[131,201],"training":[133],"to":[134,165,210],"examine":[135],"ability":[137],"unseen":[139],"queries.We":[140],"also":[141],"translated":[142],"test":[144],"set":[145],"into":[146],"English":[147],"simplified":[149],"analyse":[152],"impact":[154],"different":[156],"models.Our":[157],"findings":[158],"emphasise":[159],"need":[161],"broader":[163],"LLM":[167],"applicability":[168],"real-world":[170],"healthcare,":[171],"will":[174],"provide":[175],"dataset":[177],"evaluation":[179],"toolkit":[180],"future":[182],"research.Our":[183],"code":[186],"are":[187],"https://github.com/":[189],"zhehengluoK/eval_empec.Recent":[190],"have":[197],"demonstrated":[198],"LLMbased":[202],"Artificial":[203],"Intelligence":[204],"(AI)":[205],"providing":[207],"accurate":[208],"answers":[209],"about":[212],"world":[213],"knowledge.":[214]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}