{"id":"https://openalex.org/W4414440407","doi":"https://doi.org/10.1016/j.rineng.2025.107362","title":"Benchmarking large language models for adverse drug reaction extraction in social media and clinical texts","display_name":"Benchmarking large language models for adverse drug reaction extraction in social media and clinical texts","publication_year":2025,"publication_date":"2025-09-23","ids":{"openalex":"https://openalex.org/W4414440407","doi":"https://doi.org/10.1016/j.rineng.2025.107362"},"language":"en","primary_location":{"id":"doi:10.1016/j.rineng.2025.107362","doi":"https://doi.org/10.1016/j.rineng.2025.107362","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.rineng.2025.107362","pdf_url":null,"source":{"id":"https://openalex.org/S4210226253","display_name":"Results in Engineering","issn_l":"2590-1230","issn":["2590-1230"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","dongbi-d","jufo-1","ki-jl-1","norway-1","poland-20"],"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Results in Engineering","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1016/j.rineng.2025.107362","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093647006","display_name":"Oumayma Elbiach","orcid":"https://orcid.org/0009-0006-7020-8175","observed_orcids":["https://orcid.org/0009-0006-7020-8175"]},"institutions":[{"id":"https://openalex.org/I81605866","display_name":"Sidi Mohamed Ben Abdellah University","ror":"https://ror.org/04efg9a07","country_code":"MA","type":"education","lineage":["https://openalex.org/I81605866"]}],"countries":["MA"],"is_corresponding":true,"raw_author_name":"Oumayma Elbiach","raw_affiliation_strings":["L3IA Laboratory, Faculty of Sciences Dhar El Mahraz, Sidi Mohamed Ben Abdellah, Fez, Morocco"],"raw_orcid":"https://orcid.org/0009-0006-7020-8175","affiliations":[{"raw_affiliation_string":"L3IA Laboratory, Faculty of Sciences Dhar El Mahraz, Sidi Mohamed Ben Abdellah, Fez, Morocco","institution_ids":["https://openalex.org/I81605866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024535851","display_name":"Hanane Grissette","orcid":"https://orcid.org/0000-0002-0904-5868","observed_orcids":["https://orcid.org/0000-0002-0904-5868"]},"institutions":[{"id":"https://openalex.org/I81605866","display_name":"Sidi Mohamed Ben Abdellah University","ror":"https://ror.org/04efg9a07","country_code":"MA","type":"education","lineage":["https://openalex.org/I81605866"]}],"countries":["MA"],"is_corresponding":true,"raw_author_name":"Hanane Grissette","raw_affiliation_strings":["Laboratory of Intelligent Systems and Application, Faculty of Sciences and Technology, University Sidi Mohamed Ben Abdellah, Fez, Morocco"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Laboratory of Intelligent Systems and Application, Faculty of Sciences and Technology, University Sidi Mohamed Ben Abdellah, Fez, Morocco","institution_ids":["https://openalex.org/I81605866"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085575952","display_name":"El Habib Nfaoui","orcid":"https://orcid.org/0000-0002-5816-0897","observed_orcids":["https://orcid.org/0000-0002-5816-0897"]},"institutions":[{"id":"https://openalex.org/I81605866","display_name":"Sidi Mohamed Ben Abdellah University","ror":"https://ror.org/04efg9a07","country_code":"MA","type":"education","lineage":["https://openalex.org/I81605866"]}],"countries":["MA"],"is_corresponding":true,"raw_author_name":"El Habib Nfaoui","raw_affiliation_strings":["L3IA Laboratory, Faculty of Sciences Dhar El Mahraz, Sidi Mohamed Ben Abdellah, Fez, Morocco"],"raw_orcid":"https://orcid.org/0000-0002-5816-0897","affiliations":[{"raw_affiliation_string":"L3IA Laboratory, Faculty of Sciences Dhar El Mahraz, Sidi Mohamed Ben Abdellah, Fez, Morocco","institution_ids":["https://openalex.org/I81605866"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5024535851","https://openalex.org/A5085575952","https://openalex.org/A5093647006"],"corresponding_institution_ids":["https://openalex.org/I81605866"],"apc_list":{"value":1940,"currency":"USD","value_usd":1940},"apc_paid":{"value":1940,"currency":"USD","value_usd":1940},"fwci":9.3531,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.97648095,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"28","issue":null,"first_page":"107362","last_page":"107362"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11943","display_name":"Pharmacovigilance and Adverse Drug Reactions","score":0.9839000105857849,"subfield":{"id":"https://openalex.org/subfields/3005","display_name":"Toxicology"},"field":{"id":"https://openalex.org/fields/30","display_name":"Pharmacology, Toxicology and Pharmaceutics"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11943","display_name":"Pharmacovigilance and Adverse Drug Reactions","score":0.9839000105857849,"subfield":{"id":"https://openalex.org/subfields/3005","display_name":"Toxicology"},"field":{"id":"https://openalex.org/fields/30","display_name":"Pharmacology, Toxicology and Pharmaceutics"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.005499999970197678,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.00419999985024333,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9129999876022339},{"id":"https://openalex.org/keywords/pharmacovigilance","display_name":"pharmacovigilance","score":0.9929999709129333},{"id":"https://openalex.org/keywords/social-media-text","display_name":"social media text","score":0.7699999809265137},{"id":"https://openalex.org/keywords/clinical-narratives","display_name":"clinical text","score":0.8949999809265137},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.7419999837875366},{"id":"https://openalex.org/keywords/in-context-learning","display_name":"in-context learning","score":0.9959999918937683},{"id":"https://openalex.org/keywords/few-shot-learning","display_name":"few-shot learning","score":0.671999990940094},{"id":"https://openalex.org/keywords/gpt-4o-mini","display_name":"GPT-4o-mini","score":0.8270000219345093},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.6830000281333923},{"id":"https://openalex.org/keywords/spurious-association","display_name":"spurious association","score":0.5889999866485596}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.803600013256073},{"id":"https://openalex.org/C518677369","wikidata":"https://www.wikidata.org/wiki/Q202833","display_name":"Social media","level":2,"score":0.761900007724762},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.678600013256073},{"id":"https://openalex.org/C57658597","wikidata":"https://www.wikidata.org/wiki/Q1550789","display_name":"Pharmacovigilance","level":3,"score":0.612500011920929},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5803999900817871},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.5521000027656555},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.47519999742507935},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.45089998841285706},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.435699999332428},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43540000915527344},{"id":"https://openalex.org/C2780542314","wikidata":"https://www.wikidata.org/wiki/Q45959","display_name":"Adverse drug reaction","level":3,"score":0.36820000410079956},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.35899999737739563},{"id":"https://openalex.org/C168725872","wikidata":"https://www.wikidata.org/wiki/Q991663","display_name":"Sophistication","level":2,"score":0.3034999966621399},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.3003999888896942},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.30000001192092896},{"id":"https://openalex.org/C2993432071","wikidata":"https://www.wikidata.org/wiki/Q45959","display_name":"Drug reaction","level":3,"score":0.29820001125335693},{"id":"https://openalex.org/C2779328685","wikidata":"https://www.wikidata.org/wiki/Q1475557","display_name":"Patient safety","level":3,"score":0.29760000109672546},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.296999990940094},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.29490000009536743},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.28049999475479126},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.2759999930858612},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.27549999952316284},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.2660999894142151},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.2540999948978424}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1016/j.rineng.2025.107362","doi":"https://doi.org/10.1016/j.rineng.2025.107362","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.rineng.2025.107362","pdf_url":null,"source":{"id":"https://openalex.org/S4210226253","display_name":"Results in Engineering","issn_l":"2590-1230","issn":["2590-1230"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","dongbi-d","jufo-1","ki-jl-1","norway-1","poland-20"],"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Results in Engineering","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:609c398d08b34cf6ab37ff32bcf6d91d","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/609c398d08b34cf6ab37ff32bcf6d91d","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Results in Engineering, Vol 28, Iss , Pp 107362- (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1016/j.rineng.2025.107362","doi":"https://doi.org/10.1016/j.rineng.2025.107362","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.rineng.2025.107362","pdf_url":null,"source":{"id":"https://openalex.org/S4210226253","display_name":"Results in Engineering","issn_l":"2590-1230","issn":["2590-1230"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","dongbi-d","jufo-1","ki-jl-1","norway-1","poland-20"],"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Results in Engineering","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Good health and well-being","id":"https://metadata.un.org/sdg/3","score":0.9391000270843506}],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W2113631586","https://openalex.org/W2131546905","https://openalex.org/W2768488789","https://openalex.org/W2885195348","https://openalex.org/W2911489562","https://openalex.org/W2927462243","https://openalex.org/W2963950895","https://openalex.org/W2972736338","https://openalex.org/W3120621182","https://openalex.org/W3193492903","https://openalex.org/W4282960273","https://openalex.org/W4319600685","https://openalex.org/W4366992541","https://openalex.org/W4378072282","https://openalex.org/W4378975491","https://openalex.org/W4384564636","https://openalex.org/W4389410112","https://openalex.org/W4391577874","https://openalex.org/W4391737993","https://openalex.org/W4391940656","https://openalex.org/W4392619666","https://openalex.org/W4393187333","https://openalex.org/W4394760669","https://openalex.org/W4395045785","https://openalex.org/W4400438755","https://openalex.org/W4401023592","https://openalex.org/W4403659746","https://openalex.org/W4404636578","https://openalex.org/W4405098407","https://openalex.org/W4405244528","https://openalex.org/W4407246683","https://openalex.org/W4409190866","https://openalex.org/W4410784101","https://openalex.org/W4412800211"],"related_works":[],"abstract_inverted_index":{"Adverse":[0],"Drug":[1],"Reactions":[2],"represent":[3],"a":[4,12,39],"significant":[5,178],"threat":[6],"to":[7,43,66,208,271],"patient":[8],"safety":[9],"and":[10,25,47,74,80,94,101,106,116,123,127,146,161,168,192,203,221,229,236],"remain":[11],"critical":[13],"focus":[14],"of":[15,71,142,194],"pharmacovigilance,":[16],"yet":[17],"traditional":[18],"extraction":[19],"methods":[20],"struggle":[21],"with":[22,118,139,246],"ambiguity,":[23],"under-reporting,":[24],"noisy":[26,260],"biomedical":[27,48],"or":[28],"social":[29],"media":[30],"texts.":[31,262],"Large":[32],"Language":[33],"Models":[34],"(LLMs)":[35],"have":[36],"emerged":[37],"as":[38],"promising":[40],"alternative":[41],"due":[42],"their":[44,51,210],"contextual":[45],"understanding":[46],"pre-training,":[49],"but":[50,255],"reliability":[52],"in":[53,188,212,258],"distinguishing":[54],"true":[55],"ADRs":[56],"from":[57],"spurious":[58],"associations":[59],"is":[60],"still":[61,185],"uncertain.":[62],"This":[63],"study":[64,217],"aims":[65],"systematically":[67],"evaluate":[68],"the":[69,135,151,190,243,247,266],"effectiveness":[70,211],"state-of-the-art":[72,219],"open-":[73,220],"closed-source":[75,222],"LLMs,":[76],"including":[77],"both":[78,166],"general-purpose":[79],"domain-specific":[81,204,269],"models,":[82],"for":[83,180,224,268],"ADR":[84,182,196,225],"extraction.":[85,226],"We":[86],"benchmark":[87,113],"GPT-3.5-turbo,":[88],"GPT-4o-mini,":[89],"Microsoft":[90],"Phi-3-mini,":[91],"LLaMA-3.2-1B,":[92],"DeepSeek,":[93],"BioMistral-7B":[95],"using":[96],"two":[97,112],"complementary":[98],"approaches:":[99],"fine-tuning":[100,137,241],"in-context":[102,153,230],"learning":[103,154,231],"(zero-shot,":[104],"one-shot,":[105],"few-shot).":[107],"Experiments":[108],"are":[109,206,232],"conducted":[110],"on":[111,121,144,148,159,163,234],"datasets,":[114],"CADEC":[115,145,160,235],"SMM4H,":[117,164],"evaluation":[119],"based":[120],"strict":[122],"relaxed":[124,140,249],"precision,":[125],"recall,":[126],"F1-score":[128],"metrics.":[129],"Results":[130,252],"show":[131],"that":[132,174,199],"GPT-4o-mini":[133,240],"achieves":[134,157,242],"highest":[136,248],"performance,":[138],"F1-scores":[141],"79.06%":[143],"66.80%":[147],"SMM4H.":[149],"In":[150],"five-shot":[152],"setting,":[155],"it":[156],"65.57%":[158],"49.34%":[162],"highlighting":[165],"progress":[167,254],"ongoing":[169],"limitations.":[170],"These":[171],"findings":[172],"demonstrate":[173],"while":[175],"LLMs":[176,223],"hold":[177],"promise":[179],"enhancing":[181],"extraction,":[183],"they":[184],"face":[186],"challenges":[187,257],"capturing":[189],"complexity":[191],"variability":[193],"real-world":[195,261],"expressions,":[197],"indicating":[198],"further":[200],"model":[201],"adaptations":[202,270],"refinements":[205],"required":[207],"improve":[209],"pharmacovigilance":[213],"applications.":[214],"\u2022":[215,227,239,251,263],"The":[216],"evaluates":[218],"Fine-tuning":[228],"compared":[233],"SMM4H":[237],"datasets.":[238],"best":[244],"performance":[245],"F1-scores.":[250],"reveal":[253],"highlight":[256],"handling":[259],"Findings":[264],"emphasize":[265],"need":[267],"enhance":[272],"pharmacovigilance.":[273]},"counts_by_year":[{"year":2026,"cited_by_count":7}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2025-10-10T00:00:00"}