{"id":"https://openalex.org/W4409450873","doi":"https://doi.org/10.1016/j.ajpe.2025.101405","title":"Exploratory Assessment of GPT-4\u2032s Effectiveness in Generating Valid Exam Items in Pharmacy Education","display_name":"Exploratory Assessment of GPT-4\u2032s Effectiveness in Generating Valid Exam Items in Pharmacy Education","publication_year":2025,"publication_date":"2025-04-15","ids":{"openalex":"https://openalex.org/W4409450873","doi":"https://doi.org/10.1016/j.ajpe.2025.101405","pmid":"https://pubmed.ncbi.nlm.nih.gov/40246172"},"language":"en","primary_location":{"id":"doi:10.1016/j.ajpe.2025.101405","doi":"https://doi.org/10.1016/j.ajpe.2025.101405","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.ajpe.2025.101405","pdf_url":null,"source":{"id":"https://openalex.org/S32993367","display_name":"American Journal of Pharmaceutical Education","issn_l":"0002-9459","issn":["0002-9459","1553-6467"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-c","doyens","jufo-1","medline","norway-1","poland-70","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"American Journal of Pharmaceutical Education","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1016/j.ajpe.2025.101405","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100495891","display_name":"Benjamin Shultz","orcid":"https://orcid.org/0009-0002-4703-4286","observed_orcids":["https://orcid.org/0009-0002-4703-4286"]},"institutions":[{"id":"https://openalex.org/I39422238","display_name":"University of Illinois Chicago","ror":"https://ror.org/02mpq6x41","country_code":"US","type":"education","lineage":["https://openalex.org/I39422238"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Benjamin Shultz","raw_affiliation_strings":["University of Illinois Chicago, Retzky College of Pharmacy, Chicago, IL, USA. Electronic address: bshultz@uic.edu","University of Illinois Chicago College of Pharmacy. Electronic address: bshultz@uic.edu"],"raw_orcid":"https://orcid.org/0009-0002-4703-4286","affiliations":[{"raw_affiliation_string":"University of Illinois Chicago, Retzky College of Pharmacy, Chicago, IL, USA. Electronic address: bshultz@uic.edu","institution_ids":["https://openalex.org/I39422238"]},{"raw_affiliation_string":"University of Illinois Chicago College of Pharmacy. Electronic address: bshultz@uic.edu","institution_ids":["https://openalex.org/I39422238"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061522146","display_name":"Robert J. DiDomenico","orcid":"https://orcid.org/0000-0002-9374-8012","observed_orcids":["https://orcid.org/0000-0002-9374-8012"]},"institutions":[{"id":"https://openalex.org/I39422238","display_name":"University of Illinois Chicago","ror":"https://ror.org/02mpq6x41","country_code":"US","type":"education","lineage":["https://openalex.org/I39422238"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Robert J. DiDomenico","raw_affiliation_strings":["University of Illinois Chicago, Retzky College of Pharmacy, Chicago, IL, USA","University of Illinois Chicago College of Pharmacy. Electronic address: rdidom1@uic.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Chicago, Retzky College of Pharmacy, Chicago, IL, USA","institution_ids":["https://openalex.org/I39422238"]},{"raw_affiliation_string":"University of Illinois Chicago College of Pharmacy. Electronic address: rdidom1@uic.edu","institution_ids":["https://openalex.org/I39422238"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111025919","display_name":"Kristen L. Goliak","orcid":"https://orcid.org/0000-0003-3880-6344","observed_orcids":["https://orcid.org/0000-0003-3880-6344"]},"institutions":[{"id":"https://openalex.org/I39422238","display_name":"University of Illinois Chicago","ror":"https://ror.org/02mpq6x41","country_code":"US","type":"education","lineage":["https://openalex.org/I39422238"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kristen Goliak","raw_affiliation_strings":["University of Illinois Chicago, Retzky College of Pharmacy, Chicago, IL, USA","University of Illinois Chicago College of Pharmacy. Electronic address: kgoliak@uic.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Chicago, Retzky College of Pharmacy, Chicago, IL, USA","institution_ids":["https://openalex.org/I39422238"]},{"raw_affiliation_string":"University of Illinois Chicago College of Pharmacy. Electronic address: kgoliak@uic.edu","institution_ids":["https://openalex.org/I39422238"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5009158500","display_name":"Jeffrey J. Mucksavage","orcid":"https://orcid.org/0009-0007-5646-0443","observed_orcids":["https://orcid.org/0009-0007-5646-0443"]},"institutions":[{"id":"https://openalex.org/I39422238","display_name":"University of Illinois Chicago","ror":"https://ror.org/02mpq6x41","country_code":"US","type":"education","lineage":["https://openalex.org/I39422238"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeffrey Mucksavage","raw_affiliation_strings":["University of Illinois Chicago, Retzky College of Pharmacy, Chicago, IL, USA","University of Illinois Chicago College of Pharmacy. Electronic address: jmuck@uic.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Chicago, Retzky College of Pharmacy, Chicago, IL, USA","institution_ids":["https://openalex.org/I39422238"]},{"raw_affiliation_string":"University of Illinois Chicago College of Pharmacy. Electronic address: jmuck@uic.edu","institution_ids":["https://openalex.org/I39422238"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5100495891"],"corresponding_institution_ids":["https://openalex.org/I39422238"],"apc_list":{"value":2400,"currency":"USD","value_usd":2400},"apc_paid":{"value":2400,"currency":"USD","value_usd":2400},"fwci":2.0255,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.87854872,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"89","issue":"5","first_page":"101405","last_page":"101405"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.6743000149726868,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.6743000149726868,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.14090000092983246,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.10209999978542328,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.996999979019165},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"artificial intelligence","score":0.5450000166893005},{"id":"https://openalex.org/keywords/automated-item-generation","display_name":"automated item generation","score":0.6790000200271606},{"id":"https://openalex.org/keywords/pharmacy-education","display_name":"pharmacy education","score":0.8119999766349792},{"id":"https://openalex.org/keywords/content-validity","display_name":"content validity","score":0.9269999861717224},{"id":"https://openalex.org/keywords/item-difficulty","display_name":"item difficulty","score":0.8859999775886536},{"id":"https://openalex.org/keywords/rasch-analysis","display_name":"Rasch analysis","score":0.8109999895095825},{"id":"https://openalex.org/keywords/psychometric-properties","display_name":"psychometric properties","score":0.7680000066757202},{"id":"https://openalex.org/keywords/drug-therapy","display_name":"pharmacotherapy","score":0.5820000171661377}],"concepts":[{"id":"https://openalex.org/C104863432","wikidata":"https://www.wikidata.org/wiki/Q614304","display_name":"Pharmacy","level":2,"score":0.8045271635055542},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.5733432173728943},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4327971935272217},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.360493004322052},{"id":"https://openalex.org/C512399662","wikidata":"https://www.wikidata.org/wiki/Q3505712","display_name":"Family medicine","level":1,"score":0.2024013102054596}],"mesh":[{"descriptor_ui":"D003479","descriptor_name":"Curriculum","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003479","descriptor_name":"Curriculum","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D004512","descriptor_name":"Education, Pharmacy","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D004512","descriptor_name":"Education, Pharmacy","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D004512","descriptor_name":"Education, Pharmacy","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":true},{"descriptor_ui":"D004512","descriptor_name":"Education, Pharmacy","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":true},{"descriptor_ui":"D004521","descriptor_name":"Educational Measurement","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D004521","descriptor_name":"Educational Measurement","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D004521","descriptor_name":"Educational Measurement","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":true},{"descriptor_ui":"D004521","descriptor_name":"Educational Measurement","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D011594","descriptor_name":"Psychometrics","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D011594","descriptor_name":"Psychometrics","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D013339","descriptor_name":"Students, Pharmacy","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D013339","descriptor_name":"Students, Pharmacy","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":2,"locations":[{"id":"doi:10.1016/j.ajpe.2025.101405","doi":"https://doi.org/10.1016/j.ajpe.2025.101405","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.ajpe.2025.101405","pdf_url":null,"source":{"id":"https://openalex.org/S32993367","display_name":"American Journal of Pharmaceutical Education","issn_l":"0002-9459","issn":["0002-9459","1553-6467"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-c","doyens","jufo-1","medline","norway-1","poland-70","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"American Journal of Pharmaceutical Education","raw_type":"journal-article"},{"id":"pmid:40246172","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40246172","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"American journal of pharmaceutical education","raw_type":"Journal Article"}],"best_oa_location":{"id":"doi:10.1016/j.ajpe.2025.101405","doi":"https://doi.org/10.1016/j.ajpe.2025.101405","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.ajpe.2025.101405","pdf_url":null,"source":{"id":"https://openalex.org/S32993367","display_name":"American Journal of Pharmaceutical Education","issn_l":"0002-9459","issn":["0002-9459","1553-6467"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-c","doyens","jufo-1","medline","norway-1","poland-70","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"American Journal of Pharmaceutical Education","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.9235000014305115,"display_name":"Quality education"},{"id":"https://metadata.un.org/sdg/3","score":0.6240000128746033,"display_name":"Good health and well-being"}],"sustainable_development_goals_aurora":[{"id":"https://metadata.un.org/sdg/4","score":0.44999998807907104,"display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W1596948422","https://openalex.org/W1998623504","https://openalex.org/W2019879356","https://openalex.org/W2148623326","https://openalex.org/W2152398619","https://openalex.org/W2233859745","https://openalex.org/W2299433740","https://openalex.org/W2315260934","https://openalex.org/W2472330306","https://openalex.org/W2529450449","https://openalex.org/W2907125152","https://openalex.org/W2979726679","https://openalex.org/W3003739249","https://openalex.org/W3035248908","https://openalex.org/W3130319171","https://openalex.org/W3185341429","https://openalex.org/W4240310297","https://openalex.org/W4310993626","https://openalex.org/W4319662928","https://openalex.org/W4321649710","https://openalex.org/W4360836968","https://openalex.org/W4360891289","https://openalex.org/W4376106412","https://openalex.org/W4378472035","https://openalex.org/W4388024652","https://openalex.org/W4389274313","https://openalex.org/W4392286406","https://openalex.org/W4393343394","https://openalex.org/W4393410497","https://openalex.org/W4400487449"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W3031052312","https://openalex.org/W4389568370","https://openalex.org/W3032375762","https://openalex.org/W1995515455","https://openalex.org/W2080531066","https://openalex.org/W2069385463","https://openalex.org/W2364973063"],"abstract_inverted_index":{"OBJECTIVE:":[0],"To":[1],"evaluate":[2],"the":[3,122,188],"effectiveness":[4,189],"of":[5,134,190],"GPT-4":[6,141],"in":[7,17,150,193,199],"generating":[8,147],"valid":[9],"multiple-choice":[10],"exam":[11,55,91,148],"items":[12,31,41,49,110],"for":[13,42,196],"assessing":[14],"therapeutic":[15],"knowledge":[16],"pharmacy":[18,60,151,200],"education.":[19,201],"METHODS:":[20],"A":[21],"custom":[22],"GPT":[23],"application":[24],"was":[25],"developed":[26],"to":[27,57,70,137,171,185],"create":[28],"60":[29,76],"case-based":[30],"from":[32],"a":[33,53,99,143],"pharmacotherapy":[34],"textbook.":[35],"Nine":[36],"subject":[37],"matter":[38],"experts":[39],"reviewed":[40],"content":[43,81,149],"validity,":[44],"difficulty,":[45],"and":[46,65,95,131,161,166,175],"quality.":[47],"Valid":[48],"were":[50,68],"compiled":[51],"into":[52],"38-question":[54],"administered":[56],"46":[58],"fourth-year":[59],"students.":[61],"Classical":[62,104],"test":[63],"theory":[64],"Rasch":[66,126],"analysis":[67,106,127],"used":[69],"assess":[71],"psychometric":[72,167],"properties.":[73],"RESULTS:":[74],"Of":[75],"generated":[77],"items,":[78],"38":[79],"met":[80],"validity":[82],"requirements,":[83],"with":[84,98,180],"only":[85],"6":[86],"accepted":[87],"without":[88],"revisions.":[89],"The":[90],"demonstrated":[92],"moderate":[93],"reliability":[94],"correlated":[96],"well":[97],"prior":[100],"cumulative":[101],"therapeutics":[102],"exam.":[103],"item":[105,135,194],"revealed":[107],"that":[108,157],"most":[109],"had":[111],"acceptable":[112],"point":[113],"biserial":[114],"correlations,":[115],"though":[116],"fewer":[117],"than":[118],"half":[119],"fell":[120],"within":[121],"recommended":[123],"difficulty":[124,136],"range.":[125],"indicated":[128],"potential":[129],"multidimensionality":[130],"suboptimal":[132],"targeting":[133],"student":[138],"ability.":[139],"CONCLUSION:":[140],"offers":[142],"preliminary":[144],"step":[145],"toward":[146],"education":[152],"but":[153],"has":[154],"clear":[155],"limitations":[156],"require":[158],"further":[159,186],"investigation":[160],"validation.":[162],"Substantial":[163],"human":[164],"oversight":[165],"evaluation":[168],"are":[169],"necessary":[170],"ensure":[172],"clinical":[173],"realism":[174],"appropriate":[176],"difficulty.":[177],"Future":[178],"research":[179],"larger":[181],"samples":[182],"is":[183],"needed":[184],"validate":[187],"artificial":[191],"intelligence":[192],"generation":[195],"high-stakes":[197],"assessments":[198]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2025-10-10T00:00:00"}