{"id":"https://openalex.org/W7225828849","doi":"https://doi.org/10.1038/s41598-026-70234-w","title":"Multimodal pill image recognition in natural environments using context-aware cross-modal attention","display_name":"Multimodal pill image recognition in natural environments using context-aware cross-modal attention","publication_year":2026,"publication_date":"2026-11-11","ids":{"openalex":"https://openalex.org/W7225828849","doi":"https://doi.org/10.1038/s41598-026-70234-w"},"language":"en","primary_location":{"id":"doi:10.1038/s41598-026-70234-w","doi":"https://doi.org/10.1038/s41598-026-70234-w","is_oa":true,"landing_page_url":"https://doi.org/10.1038/s41598-026-70234-w","pdf_url":null,"source":{"id":"https://openalex.org/S196734849","display_name":"Scientific Reports","issn_l":"2045-2322","issn":["2045-2322"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-1","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319908","host_organization_name":"Nature Portfolio","host_organization_lineage":["https://openalex.org/P4310319908","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Nature Portfolio","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Scientific Reports","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1038/s41598-026-70234-w","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5161431857","display_name":"Linh Nguyen Thi My","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I3020192730","display_name":"Tr\u01b0\u1eddng \u0110H Nguy\u1ec5n T\u1ea5t Th\u00e0nh","ror":"https://ror.org/04r9s1v23","country_code":"VN","type":"education","lineage":["https://openalex.org/I3020192730"]},{"id":"https://openalex.org/I4210123993","display_name":"Van Lang University","ror":"https://ror.org/02ryrf141","country_code":"VN","type":"education","lineage":["https://openalex.org/I4210123993"]}],"countries":["VN"],"is_corresponding":false,"raw_author_name":"Linh Nguyen Thi My","raw_affiliation_strings":["Faculty of Information Technology, Nguyen Tat Thanh University, 300A Nguyen Tat Thanh Street, Xom Chieu Ward, Ho Chi Minh City, 70000, Vietnam","Faculty of Information Technology, Van Lang School of Technology, Van Lang University, 69/68 Dang Thuy Tram Street, Binh Loi Trung Ward, Ho Chi Minh City, 70000, Vietnam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Information Technology, Nguyen Tat Thanh University, 300A Nguyen Tat Thanh Street, Xom Chieu Ward, Ho Chi Minh City, 70000, Vietnam","institution_ids":["https://openalex.org/I3020192730"]},{"raw_affiliation_string":"Faculty of Information Technology, Van Lang School of Technology, Van Lang University, 69/68 Dang Thuy Tram Street, Binh Loi Trung Ward, Ho Chi Minh City, 70000, Vietnam","institution_ids":["https://openalex.org/I4210123993"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006709815","display_name":"Tham Vo","orcid":"https://orcid.org/0000-0001-7291-4168","observed_orcids":["https://orcid.org/0000-0001-7291-4168","https://orcid.org/0000-0001-8493-9004"]},"institutions":[{"id":"https://openalex.org/I3020192730","display_name":"Tr\u01b0\u1eddng \u0110H Nguy\u1ec5n T\u1ea5t Th\u00e0nh","ror":"https://ror.org/04r9s1v23","country_code":"VN","type":"education","lineage":["https://openalex.org/I3020192730"]}],"countries":["VN"],"is_corresponding":false,"raw_author_name":"Tham Vo","raw_affiliation_strings":["Faculty of Information Technology, Nguyen Tat Thanh University, 300A Nguyen Tat Thanh Street, Xom Chieu Ward, Ho Chi Minh City, 70000, Vietnam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Information Technology, Nguyen Tat Thanh University, 300A Nguyen Tat Thanh Street, Xom Chieu Ward, Ho Chi Minh City, 70000, Vietnam","institution_ids":["https://openalex.org/I3020192730"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021444959","display_name":"Vinh Truong Hoang","orcid":"https://orcid.org/0000-0002-3464-3894","observed_orcids":["https://orcid.org/0000-0002-3464-3894"]},"institutions":[{"id":"https://openalex.org/I4210096063","display_name":"Ho Chi Minh City Open University","ror":"https://ror.org/00tean533","country_code":"VN","type":"education","lineage":["https://openalex.org/I4210096063"]}],"countries":["VN"],"is_corresponding":true,"raw_author_name":"Vinh Truong Hoang","raw_affiliation_strings":["Faculty of Information Technology, Ho Chi Minh City Open University, 35-37 Ho Hao Hon Street, Cau Ong Lanh Ward, Ho Chi Minh City, 70000, Vietnam"],"raw_orcid":"https://orcid.org/0000-0002-3464-3894","affiliations":[{"raw_affiliation_string":"Faculty of Information Technology, Ho Chi Minh City Open University, 35-37 Ho Hao Hon Street, Cau Ong Lanh Ward, Ho Chi Minh City, 70000, Vietnam","institution_ids":["https://openalex.org/I4210096063"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5021444959"],"corresponding_institution_ids":["https://openalex.org/I4210096063"],"apc_list":{"value":2690,"currency":"USD","value_usd":2690},"apc_paid":{"value":2690,"currency":"USD","value_usd":2690},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.7056000232696533,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.7056000232696533,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10601","display_name":"Handwritten Text Recognition Techniques","score":0.12110000103712082,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.0763000026345253,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6776999831199646},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.6337000131607056},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6328999996185303},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5649999976158142},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.5307000279426575},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.5199999809265137},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.5148000121116638},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.5002999901771545},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.49869999289512634},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.4708000123500824},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.43709999322891235},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.4000999927520752},{"id":"https://openalex.org/C81603835","wikidata":"https://www.wikidata.org/wiki/Q3457430","display_name":"Pill","level":2,"score":0.384799987077713},{"id":"https://openalex.org/C69744172","wikidata":"https://www.wikidata.org/wiki/Q860822","display_name":"Image fusion","level":3,"score":0.37070000171661377},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.36629998683929443},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.33250001072883606},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3312000036239624},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2799000144004822},{"id":"https://openalex.org/C184297639","wikidata":"https://www.wikidata.org/wiki/Q177765","display_name":"Biometrics","level":2,"score":0.2720000147819519}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1038/s41598-026-70234-w","doi":"https://doi.org/10.1038/s41598-026-70234-w","is_oa":true,"landing_page_url":"https://doi.org/10.1038/s41598-026-70234-w","pdf_url":null,"source":{"id":"https://openalex.org/S196734849","display_name":"Scientific Reports","issn_l":"2045-2322","issn":["2045-2322"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-1","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319908","host_organization_name":"Nature Portfolio","host_organization_lineage":["https://openalex.org/P4310319908","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Nature Portfolio","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Scientific Reports","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1038/s41598-026-70234-w","doi":"https://doi.org/10.1038/s41598-026-70234-w","is_oa":true,"landing_page_url":"https://doi.org/10.1038/s41598-026-70234-w","pdf_url":null,"source":{"id":"https://openalex.org/S196734849","display_name":"Scientific Reports","issn_l":"2045-2322","issn":["2045-2322"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-1","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319908","host_organization_name":"Nature Portfolio","host_organization_lineage":["https://openalex.org/P4310319908","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Nature Portfolio","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Scientific Reports","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Medication":[0],"errors":[1],"remain":[2],"a":[3,11,14,44,52,68,85,110,287],"major":[4],"source":[5],"of":[6,88,96,149,197],"preventable":[7],"harm,":[8],"and":[9,31,84,91,104,118,141,153,159,162,279,319,330],"identifying":[10],"pill":[12],"from":[13,36,71,231],"photograph":[15],"taken":[16],"in":[17,50],"an":[18,76,236],"uncontrolled":[19],"setting":[20],"is":[21,178,220,238,260],"still":[22],"difficult":[23],"because":[24],"the":[25,40,146,171,193,223,227,248,253,277,297,306],"informative":[26],"cues-color,":[27],"shape,":[28],"surface":[29],"texture,":[30],"imprinted":[32],"text-are":[33],"unequally":[34],"reliable":[35],"one":[37],"image":[38,66],"to":[39,226,233,252,296],"next.":[41],"We":[42],"present":[43],"recognition":[45],"framework":[46],"for":[47,64,305],"natural":[48],"environments":[49],"which":[51,182],"fusion":[53,119,164,217,328],"module,":[54],"context-aware":[55],"cross-modal":[56],"attention":[57,327],"(CACMA),":[58],"weighs":[59],"these":[60,322],"four":[61],"modalities":[62],"separately":[63],"every":[65],"using":[67,273],"query":[69],"formed":[70],"their":[72],"joint":[73],"global":[74],"context,":[75],"explicit":[77],"confidence":[78,224],"branch":[79],"that":[80,300],"estimates":[81],"per-modality":[82],"reliability,":[83],"three-way":[86],"integration":[87],"attended,":[89],"confidence-weighted,":[90],"residual":[92],"features,":[93],"on":[94,128,308],"top":[95],"\\(\\hbox":[97],"{U}^2\\)":[98],"-Net":[99],"segmentation":[100,313],"(98.06%":[101],"validation":[102],"IoU)":[103],"connected-component":[105],"localization.":[106],"All":[107],"results":[108,148],"follow":[109],"unified":[111],"end-to-end":[112],"protocol,":[113],"with":[114,247,316],"segmentation,":[115],"localization,":[116],"OCR,":[117],"executed":[120],"at":[121,286],"inference":[122],"time,":[123],"over":[124,175],"three":[125],"random":[126],"seeds":[127],"two":[129],"real-world":[130],"datasets.":[131],"On":[132,205],"CURE":[133],"(196":[134],"classes)":[135,208],"CACMA":[136,191,285],"attains":[137,192],"91.13\u00b10.27%":[138],"Top-1":[139,213],"accuracy":[140],"\\(95.50\\pm":[142],"0.16\\%\\)":[143],"mAP,":[144],"exceeding":[145],"published":[147],"Ling":[150],"et":[151,155],"al.":[152,156],"R\u00e1dli":[154],"by":[157,165],"22.83":[158],"5.41":[160],"pp":[161,167,173],"summation":[163,177],"3.15":[166],"(":[168,201,240,265],"\\(p<0.001\\)":[169],");":[170],"0.76":[172],"margin":[174],"weighted":[176],"not":[179,261],"statistically":[180],"significant,":[181],"we":[183],"report":[184],"as":[185],"parity":[186],"rather":[187],"than":[188],"superiority,":[189],"while":[190,272],"smallest":[194],"across-seed":[195],"variability":[196],"any":[198],"method":[199],"evaluated":[200],"\\(\\pm":[202],"0.27\\)":[203],"pp).":[204],"OGYEI-v2":[206],"(112":[207],"it":[209,332],"reaches":[210],"\\(83.23\\pm":[211],"0.73\\%\\)":[212],"accuracy.":[214],"What":[215],"fixed-weight":[216],"cannot":[218],"reproduce":[219],"directly":[221],"measurable:":[222],"assigned":[225],"text":[228],"modality":[229],"rises":[230],"0.362":[232],"0.602":[234],"when":[235],"imprint":[237],"legible":[239],"\\(p":[241],"=":[242],"1":[243],"\\times":[244],"10^{-55}\\)":[245],"),":[246],"released":[249],"weight":[250],"redistributed":[251],"visual":[254],"modalities.":[255],"A":[256],"fully":[257],"fine-tuned":[258],"ViT-B/16":[259],"significantly":[262],"more":[263],"accurate":[264],"\\(92.07\\pm":[266],"0.64\\%\\)":[267],";":[268],"McNemar":[269],"\\(p=0.32\\)":[270],")":[271],"6.4":[274],"\\(\\times":[275],"\\)":[276],"parameters":[278],"offering":[280],"no":[281,301],"modality-level":[282],"explanation,":[283],"placing":[284],"favorable":[288],"accuracy-efficiency-interpretability":[289],"operating":[290],"point.":[291],"Paired":[292],"significance":[293],"testing":[294],"applied":[295],"ablations":[298],"shows":[299],"individual":[302],"component":[303],"accounts":[304],"result":[307],"its":[309],"own":[310],"under":[311],"this":[312],"quality;":[314],"together":[315],"calibration,":[317],"saliency":[318],"failure":[320],"analyses,":[321],"tests":[323],"delimit":[324],"where":[325,331],"multimodal":[326],"helps":[329],"does":[333],"not.":[334]},"counts_by_year":[],"updated_date":"2026-11-12T12:28:57.059296","created_date":"2026-11-12T00:00:00"}