{"id":"https://openalex.org/W4416035050","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.263","title":"LLMEval-Med: A Real-world Clinical Benchmark for Medical LLMs with Physician Validation","display_name":"LLMEval-Med: A Real-world Clinical Benchmark for Medical LLMs with Physician Validation","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416035050","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.263"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.263","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.263","pdf_url":"https://aclanthology.org/2025.findings-emnlp.263.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-emnlp.263.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100447280","display_name":"Ming Zhang","orcid":"https://orcid.org/0000-0002-5834-0994"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ming Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104237394","display_name":"Yujiong Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yujiong Shen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112454411","display_name":"Z. Li","orcid":"https://orcid.org/0009-0002-1589-0619"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zelin Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Huayu Sha","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huayu Sha","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111512931","display_name":"Binze Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Binze Hu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100330352","display_name":"Yuhui Wang","orcid":"https://orcid.org/0000-0001-8025-6724"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuhui Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059431779","display_name":"Chenhao Huang","orcid":"https://orcid.org/0000-0002-7193-9766"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chenhao Huang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076525382","display_name":"Shichun Liu","orcid":"https://orcid.org/0009-0003-0312-9846"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shichun Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113207942","display_name":"Jingqi Tong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jingqi Tong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101729628","display_name":"Changhao Jiang","orcid":"https://orcid.org/0000-0002-3098-3891"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Changhao Jiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109693861","display_name":"Mingxu Chai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mingxu Chai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036875405","display_name":"Zhiheng Xi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhiheng Xi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083173421","display_name":"Shihan Dou","orcid":"https://orcid.org/0009-0002-6013-3035"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shihan Dou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058353652","display_name":"Tao Gui","orcid":"https://orcid.org/0000-0002-6154-0751"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tao Gui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100360407","display_name":"Qi Zhang","orcid":"https://orcid.org/0000-0003-0947-4942"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5088834359","display_name":"Xuanjing Huang","orcid":"https://orcid.org/0000-0001-9197-9426"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xuanjing Huang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":16,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3295,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.85029681,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"4888","last_page":"4914"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.2653999924659729,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.2653999924659729,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.2468000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.16509999334812164,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5684999823570251},{"id":"https://openalex.org/keywords/risk-assessment","display_name":"Risk assessment","score":0.3310000002384186},{"id":"https://openalex.org/keywords/medline","display_name":"MEDLINE","score":0.32089999318122864},{"id":"https://openalex.org/keywords/public-health","display_name":"Public health","score":0.31360000371932983},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.26489999890327454}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5684999823570251},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.5590000152587891},{"id":"https://openalex.org/C545542383","wikidata":"https://www.wikidata.org/wiki/Q2751242","display_name":"Medical emergency","level":1,"score":0.37070000171661377},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.3310000002384186},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.32089999318122864},{"id":"https://openalex.org/C138816342","wikidata":"https://www.wikidata.org/wiki/Q189603","display_name":"Public health","level":2,"score":0.31360000371932983},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3057999908924103},{"id":"https://openalex.org/C512399662","wikidata":"https://www.wikidata.org/wiki/Q3505712","display_name":"Family medicine","level":1,"score":0.2842000126838684},{"id":"https://openalex.org/C177713679","wikidata":"https://www.wikidata.org/wiki/Q679690","display_name":"Intensive care medicine","level":1,"score":0.27399998903274536},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.26489999890327454},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.26179999113082886},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.259799987077713},{"id":"https://openalex.org/C142853389","wikidata":"https://www.wikidata.org/wiki/Q744778","display_name":"Association (psychology)","level":2,"score":0.2581999897956848},{"id":"https://openalex.org/C194828623","wikidata":"https://www.wikidata.org/wiki/Q2861470","display_name":"Emergency medicine","level":1,"score":0.25429999828338623},{"id":"https://openalex.org/C2779974597","wikidata":"https://www.wikidata.org/wiki/Q28448986","display_name":"Clinical Practice","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-emnlp.263","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.263","pdf_url":"https://aclanthology.org/2025.findings-emnlp.263.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.263","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.263","pdf_url":"https://aclanthology.org/2025.findings-emnlp.263.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309612","display_name":"Natural Science Foundation of Shanghai","ror":null},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320327803","display_name":"Shanghai Rising-Star Program","ror":null}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416035050.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Ming":[0],"Zhang,":[1,29],"Yujiong":[2],"Shen,":[3],"Zelin":[4],"Li,":[5],"Huayu":[6],"Sha,":[7],"Binze":[8],"Hu,":[9],"Yuhui":[10],"Wang,":[11],"Chenhao":[12],"Huang,":[13],"Shichun":[14],"Liu,":[15],"Jingqi":[16],"Tong,":[17],"Changhao":[18],"Jiang,":[19],"Mingxu":[20],"Chai,":[21],"Zhiheng":[22],"Xi,":[23],"Shihan":[24],"Dou,":[25],"Tao":[26],"Gui,":[27],"Qi":[28],"Xuanjing":[30],"Huang.":[31],"Findings":[32],"of":[33],"the":[34],"Association":[35],"for":[36],"Computational":[37],"Linguistics:":[38],"EMNLP":[39],"2025.":[40,41]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-11-08T00:00:00"}