{"id":"https://openalex.org/W7225408995","doi":"https://doi.org/10.2196/preprints.116306","title":"Linguistic Markers of Error and Faculty-Validated Model Rating of Artificial Intelligence Explanations in Pharmacy Education: Development and Validation Study (Preprint)","display_name":"Linguistic Markers of Error and Faculty-Validated Model Rating of Artificial Intelligence Explanations in Pharmacy Education: Development and Validation Study (Preprint)","publication_year":2026,"publication_date":"2026-11-06","ids":{"openalex":"https://openalex.org/W7225408995","doi":"https://doi.org/10.2196/preprints.116306"},"language":"en","primary_location":{"id":"doi:10.2196/preprints.116306","doi":"https://doi.org/10.2196/preprints.116306","is_oa":false,"landing_page_url":"https://doi.org/10.2196/preprints.116306","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5161039522","display_name":"Seungjun Kim","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Seungjun Kim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160989991","display_name":"Andre Fabre","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Andre Fabre","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004046753","display_name":"S.J. Ramon L. Bautista","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ramon Bautista","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160992888","display_name":"Jordan An Thinh Dinh","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jordan An Thinh Dinh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063330851","display_name":"Brandon Fernandez","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Brandon Fernandez","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5161001477","display_name":"Justin Kwong","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Justin Kwong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005352486","display_name":"Krystal Nguyen","orcid":"https://orcid.org/0009-0000-7312-3974","observed_orcids":["https://orcid.org/0009-0000-7312-3974"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krystal Nguyen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077694994","display_name":"Nahyun Mariah Kim","orcid":"https://orcid.org/0000-0002-1751-1275","observed_orcids":["https://orcid.org/0000-0002-1751-1275"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nahyun Mariah Kim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020234173","display_name":"Shawn Patrick Griffin","orcid":"https://orcid.org/0000-0002-3818-2490","observed_orcids":["https://orcid.org/0000-0002-3818-2490"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shawn P Griffin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012631074","display_name":"Keri Hurley","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Keri Hurley","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5161085456","display_name":"Farah Khorassani","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Farah Khorassani","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027396222","display_name":"Yael Marks","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yael Marks","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5020791705","display_name":"Ghahyun J. Kim","orcid":"https://orcid.org/0000-0001-9073-2034","observed_orcids":["https://orcid.org/0000-0001-9073-2034"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gha-hyun Jeffrey Kim","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-9073-2034","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9599999785423279,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9599999785423279,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.011599999852478504,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.006200000178068876,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9470000267028809},{"id":"https://openalex.org/keywords/llm-generated-explanations","display_name":"LLM-generated explanations","score":0.6359999775886536},{"id":"https://openalex.org/keywords/explanation-quality","display_name":"explanation quality","score":0.671999990940094},{"id":"https://openalex.org/keywords/error-detection","display_name":"error detection","score":0.49300000071525574},{"id":"https://openalex.org/keywords/pharmacy-education","display_name":"pharmacy education","score":0.5709999799728394},{"id":"https://openalex.org/keywords/response-reliability","display_name":"response reliability","score":0.5109999775886536},{"id":"https://openalex.org/keywords/chatgpt","display_name":"ChatGPT","score":0.6880000233650208},{"id":"https://openalex.org/keywords/gemini","display_name":"Gemini","score":0.9559999704360962},{"id":"https://openalex.org/keywords/keyness-analysis","display_name":"keyness analysis","score":0.49000000953674316},{"id":"https://openalex.org/keywords/response-styles","display_name":"response styles","score":0.38199999928474426},{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.3140000104904175}],"concepts":[{"id":"https://openalex.org/C186282968","wikidata":"https://www.wikidata.org/wiki/Q1434261","display_name":"McNemar's test","level":2,"score":0.6870999932289124},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5461999773979187},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5418999791145325},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5263000130653381},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5242000222206116},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5184999704360962},{"id":"https://openalex.org/C104863432","wikidata":"https://www.wikidata.org/wiki/Q614304","display_name":"Pharmacy","level":2,"score":0.46209999918937683},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.43160000443458557},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.38659998774528503},{"id":"https://openalex.org/C143095724","wikidata":"https://www.wikidata.org/wiki/Q515895","display_name":"Odds","level":3,"score":0.36329999566078186},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.35420000553131104},{"id":"https://openalex.org/C2781234862","wikidata":"https://www.wikidata.org/wiki/Q6543077","display_name":"Licensure","level":2,"score":0.34209999442100525},{"id":"https://openalex.org/C2778780117","wikidata":"https://www.wikidata.org/wiki/Q3269423","display_name":"Reading comprehension","level":3,"score":0.3407000005245209},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.32850000262260437},{"id":"https://openalex.org/C2780957641","wikidata":"https://www.wikidata.org/wiki/Q1999796","display_name":"Spell","level":2,"score":0.32670000195503235},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.32339999079704285},{"id":"https://openalex.org/C554936623","wikidata":"https://www.wikidata.org/wiki/Q199657","display_name":"Reading (process)","level":2,"score":0.32280001044273376},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3131999969482422},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.31209999322891235},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.31200000643730164},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3109000027179718},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.29249998927116394},{"id":"https://openalex.org/C2777146004","wikidata":"https://www.wikidata.org/wiki/Q14949826","display_name":"CLARITY","level":2,"score":0.290800005197525},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.28630000352859497},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.2685000002384186},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.266400009393692},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.26510000228881836},{"id":"https://openalex.org/C2779077324","wikidata":"https://www.wikidata.org/wiki/Q1087138","display_name":"Dictation","level":2,"score":0.2614000141620636},{"id":"https://openalex.org/C46304622","wikidata":"https://www.wikidata.org/wiki/Q374814","display_name":"Certification","level":2,"score":0.2597000002861023},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.25859999656677246},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2583000063896179},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2574000060558319},{"id":"https://openalex.org/C106906290","wikidata":"https://www.wikidata.org/wiki/Q1140909","display_name":"Cronbach's alpha","level":3,"score":0.2533999979496002},{"id":"https://openalex.org/C70364389","wikidata":"https://www.wikidata.org/wiki/Q18757","display_name":"Validity","level":3,"score":0.25110000371932983},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.25029999017715454}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.2196/preprints.116306","doi":"https://doi.org/10.2196/preprints.116306","is_oa":false,"landing_page_url":"https://doi.org/10.2196/preprints.116306","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6164000034332275,"display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W1507711477","https://openalex.org/W2020018978","https://openalex.org/W2037789405","https://openalex.org/W2038411619","https://openalex.org/W2084413241","https://openalex.org/W2164777277","https://openalex.org/W2244501064","https://openalex.org/W4309674289","https://openalex.org/W4319460874","https://openalex.org/W4384071683","https://openalex.org/W4386117408","https://openalex.org/W4386510404","https://openalex.org/W4392196456","https://openalex.org/W4401043020","https://openalex.org/W4402671701","https://openalex.org/W4411038165","https://openalex.org/W4417318012","https://openalex.org/W7133239970"],"related_works":["https://openalex.org/W4406228650","https://openalex.org/W4402692392","https://openalex.org/W7216072963","https://openalex.org/W7114913595","https://openalex.org/W4393155265","https://openalex.org/W4411223300","https://openalex.org/W7163400639","https://openalex.org/W4410594013","https://openalex.org/W4405523614","https://openalex.org/W7202358203"],"abstract_inverted_index":{"BACKGROUND":[0],"When":[1],"a":[2,6,12,37,40,73,172,184,240,245,248,321,324,363,374,406],"pharmacy":[3,95],"student":[4,26],"asks":[5],"large":[7],"language":[8,62,74,185],"model":[9,75,186,279,375],"(LLM)":[10],"tool":[11],"question,":[13,266],"it":[14,415],"returns":[15],"an":[16,20,271,349],"answer":[17,33,135],"and":[18,24,49,71,92,102,119,137,155,162,171,183,197,215,238,244,327,334,336,362,373,385,405,421],"usually":[19],"explanation":[21,82,100,337,392,429],"of":[22,64,81,210,255,276,315,348,355,423],"why,":[23],"the":[25,29,134,194,253,262,265,274,302,340,366,417,428,433],"learns":[27],"from":[28,46,221],"explanation.":[30],"A":[31,258],"correct":[32,317],"can":[34,67,77,394,413],"sit":[35],"on":[36,94,122,193,228,301,380],"wrong":[38,69,234,268],"explanation,":[39],"response\u2019s":[41],"reliability":[42],"is":[43],"rarely":[44],"apparent":[45],"reading":[47],"it,":[48],"students":[50,419],"have":[51],"no":[52],"dependable":[53],"way":[54],"to":[55,88,160,163,188,223,295,299,308,370,416],"tell.":[56],"OBJECTIVE":[57],"We":[58,129],"tested":[59],"whether":[60,72],"natural":[61],"processing":[63],"LLM":[65,206,350],"responses":[66,132],"identify":[68],"answers":[70,235,269,318,367],"rater":[76,280,376],"reproduce":[78],"faculty":[79,164,178,195,283,287,378],"judgments":[80],"quality,":[83,101,384],"then":[84],"applied":[85],"these":[86,424],"methods":[87],"Liner":[89],"Scholar,":[90],"ChatGPT,":[91],"Gemini":[93],"questions":[96,116,125,212],"across":[97,388],"accuracy,":[98],"stability,":[99],"response":[103,246,351,403],"style.":[104],"METHODS":[105],"Each":[106],"product":[107,189],"answered":[108,208],"248":[109,211],"investigator-developed":[110],"North":[111],"American":[112],"Pharmacist":[113],"Licensure":[114],"Examination-style":[115],"(744":[117],"responses),":[118],"two":[120],"experiments":[121],"24":[123],"adapted":[124],"added":[126],"792":[127],"responses.":[128,310],"scored":[130],"benchmark":[131],"against":[133,200,297],"key":[136],"compared":[138],"products":[139],"using":[140,260],"exact":[141],"McNemar":[142],"tests.":[143],"Fixed":[144,359],"linguistic":[145,360],"rules":[146,361],"measured":[147],"length,":[148],"source":[149,241,249],"references,":[150],"hedging,":[151],"boosters,":[152],"distractor":[153],"coverage,":[154],"readability,":[156],"which":[157],"were":[158,236,330],"related":[159],"correctness":[161],"ratings":[165],"in":[166,342],"regression":[167],"models,":[168],"keyness":[169],"analysis,":[170],"cross":[173],"validated":[174,199],"text":[175,347,404],"classifier.":[176],"Four":[177],"had":[179,250],"rated":[180,190,331],"111":[181],"responses,":[182,313],"blinded":[187],"all":[191],"744":[192,309],"codebook":[196],"was":[198],"those":[201,312],"ratings.":[202],"RESULTS":[203],"The":[204,278,346,399],"three":[205,303],"tools":[207,341,418,425],"210":[209],"correctly":[213],"together":[214],"missed":[216],"8":[217],"together,":[218],"with":[219,270,282,289,323],"accuracy":[220,335],"87.5%":[222],"95.6%,":[224],"so":[225,391,408],"differences":[226],"rested":[227],"30":[229],"questions.":[230],"In":[231],"every":[232,389],"product,":[233],"longer":[237],"named":[239],"more":[242],"often,":[243],"naming":[247],"0.41":[251],"times":[252],"odds":[254],"being":[256],"correct.":[257],"classifier":[259,364],"only":[261,402],"response,":[263,390],"without":[264],"identified":[267,365],"area":[272],"under":[273],"curve":[275],"0.79.":[277],"agreed":[281,288],"as":[284,286,430,432],"closely":[285],"each":[290],"other":[291],"(weighted":[292],"kappa":[293],"0.53":[294],"0.77":[296],"0.51":[298],"0.67":[300],"ordinal":[304],"variables),":[305],"extending":[306],"review":[307],"Across":[311],"63":[314],"680":[316],"(9.3%)":[319],"carried":[320],"rationale":[322,381],"meaningful":[325],"error":[326],"42":[328],"(6.2%)":[329],"educationally":[332],"poor,":[333],"quality":[338,393],"ranked":[339],"different":[343],"orders.":[344],"CONCLUSIONS":[345],"carries":[352],"measurable":[353],"evidence":[354],"its":[356],"own":[357],"reliability.":[358],"most":[368],"likely":[369],"be":[371,395],"wrong,":[372],"reproduced":[377],"judgment":[379],"correctness,":[382],"educational":[383],"clinical":[386],"safety":[387],"audited":[396],"at":[397],"scale.":[398],"approach":[400],"needs":[401],"rubric,":[407],"any":[409],"health":[410],"professions":[411],"program":[412],"apply":[414],"use,":[420],"evaluation":[422],"should":[426],"score":[427],"well":[431],"answer.":[434]},"counts_by_year":[],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2026-11-07T00:00:00"}