{"id":"https://openalex.org/W7138940159","doi":"https://doi.org/10.1609/aaai.v40i40.40731","title":"EHRStruct: A Comprehensive Benchmark Framework for Evaluating Large Language Models on Structured Electronic Health Record Tasks","display_name":"EHRStruct: A Comprehensive Benchmark Framework for Evaluating Large Language Models on Structured Electronic Health Record Tasks","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7138940159","doi":"https://doi.org/10.1609/aaai.v40i40.40731"},"language":null,"primary_location":{"id":"doi:10.1609/aaai.v40i40.40731","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i40.40731","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40731/44692","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40731/44692","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5130186601","display_name":"Xiao Yang","orcid":null},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":true,"raw_author_name":"Xiao Yang","raw_affiliation_strings":["Nanyang Technological University"],"affiliations":[{"raw_affiliation_string":"Nanyang Technological University","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130124639","display_name":"Xuejiao Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Xuejiao Zhao","raw_affiliation_strings":["Nanyang Technological University"],"affiliations":[{"raw_affiliation_string":"Nanyang Technological University","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5129779996","display_name":"Zhiqi Shen","orcid":null},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Zhiqi Shen","raw_affiliation_strings":["Nanyang Technological University"],"affiliations":[{"raw_affiliation_string":"Nanyang Technological University","institution_ids":["https://openalex.org/I172675005"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5130186601"],"corresponding_institution_ids":["https://openalex.org/I172675005"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.85974304,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"40","first_page":"34340","last_page":"34348"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.965399980545044,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.965399980545044,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.016300000250339508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10350","display_name":"Electronic Health Records Systems","score":0.006899999920278788,"subfield":{"id":"https://openalex.org/subfields/3605","display_name":"Health Information Management"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6528000235557556},{"id":"https://openalex.org/keywords/electronic-health-record","display_name":"Electronic health record","score":0.5595999956130981},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5329999923706055},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5328999757766724},{"id":"https://openalex.org/keywords/health-records","display_name":"Health records","score":0.43630000948905945},{"id":"https://openalex.org/keywords/medical-record","display_name":"Medical record","score":0.3215999901294708},{"id":"https://openalex.org/keywords/relational-database","display_name":"Relational database","score":0.31850001215934753}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7716000080108643},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6528000235557556},{"id":"https://openalex.org/C3020144179","wikidata":"https://www.wikidata.org/wiki/Q10871684","display_name":"Electronic health record","level":3,"score":0.5595999956130981},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5329999923706055},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5328999757766724},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5163999795913696},{"id":"https://openalex.org/C3019952477","wikidata":"https://www.wikidata.org/wiki/Q1324077","display_name":"Health records","level":3,"score":0.43630000948905945},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3783999979496002},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3246999979019165},{"id":"https://openalex.org/C195910791","wikidata":"https://www.wikidata.org/wiki/Q1324077","display_name":"Medical record","level":2,"score":0.3215999901294708},{"id":"https://openalex.org/C5655090","wikidata":"https://www.wikidata.org/wiki/Q192588","display_name":"Relational database","level":2,"score":0.31850001215934753},{"id":"https://openalex.org/C3018060332","wikidata":"https://www.wikidata.org/wiki/Q10871684","display_name":"Electronic medical record","level":2,"score":0.3183000087738037},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3158999979496002},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2957000136375427},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.29010000824928284},{"id":"https://openalex.org/C145642194","wikidata":"https://www.wikidata.org/wiki/Q870895","display_name":"Health informatics","level":3,"score":0.288100004196167},{"id":"https://openalex.org/C3019659195","wikidata":"https://www.wikidata.org/wiki/Q5690566","display_name":"Meaningful use","level":3,"score":0.287200003862381},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.27250000834465027},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.2671000063419342},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2599000036716461}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1609/aaai.v40i40.40731","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i40.40731","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40731/44692","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i40.40731","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i40.40731","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40731/44692","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.7923343181610107}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7138940159.pdf","grobid_xml":"https://content.openalex.org/works/W7138940159.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Structured":[0],"Electronic":[1],"Health":[2],"Record":[3],"(EHR)":[4],"data":[5,150],"stores":[6],"patient":[7],"information":[8],"in":[9,17],"relational":[10],"tables":[11],"and":[12,48,58,94,114,120,136,139,166,183],"plays":[13],"a":[14,74,176],"central":[15],"role":[16],"clinical":[18,39,92],"decision-making.":[19],"Recent":[20],"advances":[21],"have":[22],"explored":[23],"the":[24,42,164],"use":[25,108],"of":[26,44,169],"large":[27],"language":[28],"models":[29],"(LLMs)":[30],"to":[31,55,78,110,187],"process":[32],"such":[33],"data,":[34],"showing":[35],"promise":[36],"across":[37],"various":[38],"tasks.":[40,84],"However,":[41],"absence":[43],"standardized":[45],"evaluation":[46,69,98],"frameworks":[47],"clearly":[49],"defined":[50],"tasks":[51,89,159],"makes":[52],"it":[53],"difficult":[54],"systematically":[56],"assess":[57],"compare":[59,140],"LLM":[60],"performance":[61,182],"on":[62,81,163],"structured":[63,82,149,157],"EHR":[64,83,105,158],"data.":[65],"To":[66],"address":[67],"these":[68],"challenges,":[70],"we":[71,173],"introduce":[72],"EHRStruct,":[73],"benchmark":[75],"specifically":[76],"designed":[77],"evaluate":[79,111],"LLMs":[80],"EHRStruct":[85,109],"defines":[86],"11":[87,143],"representative":[88,115],"spanning":[90],"diverse":[91],"needs":[93],"includes":[95],"2,200":[96],"task-specific":[97],"samples":[99],"derived":[100],"from":[101],"two":[102],"widely":[103],"used":[104],"datasets.":[106],"We":[107,123],"20":[112],"advanced":[113],"LLMs,":[116],"covering":[117],"both":[118],"general":[119],"medical":[121],"models.":[122],"further":[124],"analyze":[125],"key":[126],"factors":[127],"influencing":[128],"model":[129],"performance,":[130],"including":[131],"input":[132],"formats,":[133],"few-shot":[134],"generalisation,":[135],"finetuning":[137],"strategies,":[138],"results":[141,153],"with":[142],"state-of-the-art":[144,181],"LLM-based":[145],"enhancement":[146],"methods":[147],"for":[148],"reasoning.":[151],"Our":[152],"indicate":[154],"that":[155,179],"many":[156],"place":[160],"high":[161],"demands":[162],"understanding":[165],"reasoning":[167],"capabilities":[168],"LLMs.":[170],"In":[171],"response,":[172],"propose":[174],"SEMaster,":[175],"code-augmented":[177],"method":[178],"achieves":[180],"offers":[184],"practical":[185],"insights":[186],"guide":[188],"future":[189],"research.":[190]},"counts_by_year":[],"updated_date":"2026-03-20T20:54:20.808490","created_date":"2026-03-20T00:00:00"}