{"id":"https://openalex.org/W4413050476","doi":"https://doi.org/10.3233/shti250995","title":"LLM-Based Medical Document Evaluation: Integrating Human Expert Insights","display_name":"LLM-Based Medical Document Evaluation: Integrating Human Expert Insights","publication_year":2025,"publication_date":"2025-08-07","ids":{"openalex":"https://openalex.org/W4413050476","doi":"https://doi.org/10.3233/shti250995","pmid":"https://pubmed.ncbi.nlm.nih.gov/40776013"},"language":"en","primary_location":{"id":"doi:10.3233/shti250995","is_oa":true,"landing_page_url":"https://doi.org/10.3233/shti250995","pdf_url":null,"source":{"id":"https://openalex.org/S4210179765","display_name":"Studies in health technology and informatics","issn_l":"0926-9630","issn":["0926-9630","1879-8365"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318577","host_organization_name":"IOS Press","host_organization_lineage":["https://openalex.org/P4310318577"],"host_organization_lineage_names":["IOS Press"],"type":"book series"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Studies in Health Technology and Informatics","raw_type":"book-chapter"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.3233/shti250995","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078101101","display_name":"Junhyuk Seo","orcid":"https://orcid.org/0009-0006-8289-4432"},"institutions":[{"id":"https://openalex.org/I2802194831","display_name":"Samsung Medical Center","ror":"https://ror.org/05a15z872","country_code":"KR","type":"healthcare","lineage":["https://openalex.org/I2250650973","https://openalex.org/I2802194831"]},{"id":"https://openalex.org/I848706","display_name":"Sungkyunkwan University","ror":"https://ror.org/04q78tk20","country_code":"KR","type":"education","lineage":["https://openalex.org/I848706"]}],"countries":["KR"],"is_corresponding":true,"raw_author_name":"Junhyuk Seo","raw_affiliation_strings":["Department of Nursing, Samsung Medical Center","SAIHST, Sungkyunkwan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Nursing, Samsung Medical Center","institution_ids":["https://openalex.org/I2802194831"]},{"raw_affiliation_string":"SAIHST, Sungkyunkwan University","institution_ids":["https://openalex.org/I848706"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113082664","display_name":"Dasol Choi","orcid":null},"institutions":[{"id":"https://openalex.org/I193775966","display_name":"Yonsei University","ror":"https://ror.org/01wjejq96","country_code":"KR","type":"education","lineage":["https://openalex.org/I193775966"]},{"id":"https://openalex.org/I3130844761","display_name":"MODUL University Dubai","ror":"https://ror.org/04rrhkc58","country_code":"AE","type":"education","lineage":["https://openalex.org/I150545927","https://openalex.org/I3130844761"]}],"countries":["AE","KR"],"is_corresponding":false,"raw_author_name":"Dasol Choi","raw_affiliation_strings":["MODULABS","Yonsei University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MODULABS","institution_ids":["https://openalex.org/I3130844761"]},{"raw_affiliation_string":"Yonsei University","institution_ids":["https://openalex.org/I193775966"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110148990","display_name":"Wonchul Cha","orcid":null},"institutions":[{"id":"https://openalex.org/I2802194831","display_name":"Samsung Medical Center","ror":"https://ror.org/05a15z872","country_code":"KR","type":"healthcare","lineage":["https://openalex.org/I2250650973","https://openalex.org/I2802194831"]},{"id":"https://openalex.org/I848706","display_name":"Sungkyunkwan University","ror":"https://ror.org/04q78tk20","country_code":"KR","type":"education","lineage":["https://openalex.org/I848706"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Wonchul Cha","raw_affiliation_strings":["Department of Emergency Medicine, Samsung Medical Center, Sungkyunkwan University School of Medicine","SAIHST, Sungkyunkwan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Emergency Medicine, Samsung Medical Center, Sungkyunkwan University School of Medicine","institution_ids":["https://openalex.org/I2802194831","https://openalex.org/I848706"]},{"raw_affiliation_string":"SAIHST, Sungkyunkwan University","institution_ids":["https://openalex.org/I848706"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076610667","display_name":"Taerim Kim","orcid":null},"institutions":[{"id":"https://openalex.org/I2802194831","display_name":"Samsung Medical Center","ror":"https://ror.org/05a15z872","country_code":"KR","type":"healthcare","lineage":["https://openalex.org/I2250650973","https://openalex.org/I2802194831"]},{"id":"https://openalex.org/I848706","display_name":"Sungkyunkwan University","ror":"https://ror.org/04q78tk20","country_code":"KR","type":"education","lineage":["https://openalex.org/I848706"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Taerim Kim","raw_affiliation_strings":["Department of Emergency Medicine, Samsung Medical Center, Sungkyunkwan University School of Medicine","SAIHST, Sungkyunkwan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Emergency Medicine, Samsung Medical Center, Sungkyunkwan University School of Medicine","institution_ids":["https://openalex.org/I2802194831","https://openalex.org/I848706"]},{"raw_affiliation_string":"SAIHST, Sungkyunkwan University","institution_ids":["https://openalex.org/I848706"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5078101101"],"corresponding_institution_ids":["https://openalex.org/I2802194831","https://openalex.org/I848706"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.34732496,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"329","issue":null,"first_page":"1029","last_page":"1033"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9466999769210815,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9466999769210815,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.647546648979187},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3903637230396271}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.647546648979187},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3903637230396271}],"mesh":[{"descriptor_ui":"D004282","descriptor_name":"Documentation","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D004282","descriptor_name":"Documentation","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D004282","descriptor_name":"Documentation","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":true},{"descriptor_ui":"D004282","descriptor_name":"Documentation","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":true},{"descriptor_ui":"D005103","descriptor_name":"Expert Systems","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D005103","descriptor_name":"Expert Systems","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D011381","descriptor_name":"Programming Languages","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D011381","descriptor_name":"Programming Languages","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057286","descriptor_name":"Electronic Health Records","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D057286","descriptor_name":"Electronic Health Records","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":2,"locations":[{"id":"doi:10.3233/shti250995","is_oa":true,"landing_page_url":"https://doi.org/10.3233/shti250995","pdf_url":null,"source":{"id":"https://openalex.org/S4210179765","display_name":"Studies in health technology and informatics","issn_l":"0926-9630","issn":["0926-9630","1879-8365"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318577","host_organization_name":"IOS Press","host_organization_lineage":["https://openalex.org/P4310318577"],"host_organization_lineage_names":["IOS Press"],"type":"book series"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Studies in Health Technology and Informatics","raw_type":"book-chapter"},{"id":"pmid:40776013","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40776013","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Studies in health technology and informatics","raw_type":null}],"best_oa_location":{"id":"doi:10.3233/shti250995","is_oa":true,"landing_page_url":"https://doi.org/10.3233/shti250995","pdf_url":null,"source":{"id":"https://openalex.org/S4210179765","display_name":"Studies in health technology and informatics","issn_l":"0926-9630","issn":["0926-9630","1879-8365"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318577","host_organization_name":"IOS Press","host_organization_lineage":["https://openalex.org/P4310318577"],"host_organization_lineage_names":["IOS Press"],"type":"book series"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Studies in Health Technology and Informatics","raw_type":"book-chapter"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"show":[4],"potential":[5],"in":[6,114],"medical":[7,112],"document":[8],"generation,":[9],"but":[10],"ensuring":[11],"reliability":[12],"requires":[13],"extensive":[14],"expert":[15,75,105],"involvement,":[16],"limiting":[17],"clinical":[18,115],"applications.":[19],"To":[20],"address":[21],"this":[22],"challenge,":[23],"we":[24],"developed":[25],"an":[26],"LLM-based":[27,99],"evaluation":[28,52,100],"framework":[29,49],"with":[30,74,104],"three":[31],"progressive":[32],"Chain":[33],"of":[34],"Thought":[35],"(CoT)":[36],"strategies:":[37],"Qualitative":[38,85],"(expert":[39,46],"persona),":[40],"Quantitative-qualitative":[41,90],"(error":[42],"analysis),":[43],"and":[44,89],"Insight-integrated":[45,69],"reasoning).":[47],"This":[48],"captures":[50],"nuanced":[51],"patterns":[53],"while":[54],"maintaining":[55],"efficiency.":[56],"When":[57],"tested":[58],"on":[59],"33":[60],"LLM-generated":[61],"Emergency":[62],"Department":[63],"records":[64],"across":[65],"five":[66],"criteria,":[67],"our":[68],"approach":[70],"demonstrated":[71],"strong":[72],"correlation":[73],"evaluations":[76],"(r":[77,86,91],"=":[78,87,92],"0.680,":[79],"p":[80],"<":[81],".001),":[82],"outperforming":[83],"both":[84],"0.524)":[88],"0.630)":[93],"approaches.":[94],"Our":[95],"findings":[96],"suggest":[97],"that":[98],"frameworks":[101],"can":[102],"align":[103],"assessments":[106],"as":[107],"useful":[108],"tools":[109],"for":[110],"validating":[111],"documentation":[113],"settings.":[116]},"counts_by_year":[],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}