{"id":"https://openalex.org/W7133308114","doi":"https://doi.org/10.7717/peerj-cs.3629","title":"Cross-modal emotion recognition with causality inference in human conversations","display_name":"Cross-modal emotion recognition with causality inference in human conversations","publication_year":2026,"publication_date":"2026-03-03","ids":{"openalex":"https://openalex.org/W7133308114","doi":"https://doi.org/10.7717/peerj-cs.3629"},"language":"en","primary_location":{"id":"doi:10.7717/peerj-cs.3629","is_oa":true,"landing_page_url":"https://doi.org/10.7717/peerj-cs.3629","pdf_url":null,"source":{"id":"https://openalex.org/S4210178049","display_name":"PeerJ Computer Science","issn_l":"2376-5992","issn":["2376-5992"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320104","host_organization_name":"PeerJ, Inc.","host_organization_lineage":["https://openalex.org/P4310320104"],"host_organization_lineage_names":["PeerJ, Inc."],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PeerJ Computer Science","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.7717/peerj-cs.3629","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041891989","display_name":"Tahani Jaser Alahmadi","orcid":"https://orcid.org/0000-0002-0067-692X"},"institutions":[{"id":"https://openalex.org/I106778892","display_name":"Princess Nourah bint Abdulrahman University","ror":"https://ror.org/05b0cyh02","country_code":"SA","type":"education","lineage":["https://openalex.org/I106778892"]}],"countries":["SA"],"is_corresponding":true,"raw_author_name":"Tahani Jaser Alahmadi","raw_affiliation_strings":["Department of Information Systems, College of Computer and Information Sciences, Princess Nourah Bint Abdulrahman University, Riyadh, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0002-0067-692X","affiliations":[{"raw_affiliation_string":"Department of Information Systems, College of Computer and Information Sciences, Princess Nourah Bint Abdulrahman University, Riyadh, Saudi Arabia","institution_ids":["https://openalex.org/I106778892"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127874676","display_name":"Galiya Ybytayeva","orcid":null},"institutions":[{"id":"https://openalex.org/I105208703","display_name":"Almaty Management University","ror":"https://ror.org/00rhgqg04","country_code":"KZ","type":"education","lineage":["https://openalex.org/I105208703"]},{"id":"https://openalex.org/I4210156956","display_name":"T.K. Zhurgenov Kazakh National Academy of Arts","ror":"https://ror.org/05jra0838","country_code":"KZ","type":"education","lineage":["https://openalex.org/I4210156956"]}],"countries":["KZ"],"is_corresponding":false,"raw_author_name":"Galiya Ybytayeva","raw_affiliation_strings":["School of Engineering, International Educational Corporation, Kazakh Leading Academy of Architecture and Civil Engineering, Almaty, Kazakhstan"],"raw_orcid":"https://orcid.org/0000-0002-4243-0928","affiliations":[{"raw_affiliation_string":"School of Engineering, International Educational Corporation, Kazakh Leading Academy of Architecture and Civil Engineering, Almaty, Kazakhstan","institution_ids":["https://openalex.org/I105208703","https://openalex.org/I4210156956"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127912934","display_name":"Akbayan Bekarystankyzy","orcid":null},"institutions":[{"id":"https://openalex.org/I3130577743","display_name":"Narxoz University","ror":"https://ror.org/0523w7v09","country_code":"KZ","type":"education","lineage":["https://openalex.org/I3130577743"]}],"countries":["KZ"],"is_corresponding":false,"raw_author_name":"Akbayan Bekarystankyzy","raw_affiliation_strings":["School of Digital Technologies, Narxoz University, Almaty, Kazakhstan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Digital Technologies, Narxoz University, Almaty, Kazakhstan","institution_ids":["https://openalex.org/I3130577743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127922716","display_name":"Khalid J. Alzahrani","orcid":null},"institutions":[{"id":"https://openalex.org/I179331831","display_name":"Taif University","ror":"https://ror.org/014g1a453","country_code":"SA","type":"education","lineage":["https://openalex.org/I179331831"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Khalid J. Alzahrani","raw_affiliation_strings":["Department of Clinical Laboratories Sciences, College of Applied Medical Sciences, Taif University, Taif, Saudi Arabia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Clinical Laboratories Sciences, College of Applied Medical Sciences, Taif University, Taif, Saudi Arabia","institution_ids":["https://openalex.org/I179331831"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016140384","display_name":"Rizwan Abbas","orcid":"https://orcid.org/0000-0001-8324-9175"},"institutions":[{"id":"https://openalex.org/I168879160","display_name":"Zhejiang University of Science and Technology","ror":"https://ror.org/05mx0wr29","country_code":"CN","type":"education","lineage":["https://openalex.org/I168879160"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rizwan Abbas","raw_affiliation_strings":["College of Computer Science and Technology, Zhejiang University, Hangzhou, Zhejiang, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Hangzhou, Zhejiang, China","institution_ids":["https://openalex.org/I168879160"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112372747","display_name":"Hala Abdelhameed","orcid":null},"institutions":[{"id":"https://openalex.org/I23075662","display_name":"Taibah University","ror":"https://ror.org/01xv1nn60","country_code":"SA","type":"education","lineage":["https://openalex.org/I23075662"]},{"id":"https://openalex.org/I66513531","display_name":"Fayoum University","ror":"https://ror.org/023gzwx10","country_code":"EG","type":"education","lineage":["https://openalex.org/I66513531"]}],"countries":["EG","SA"],"is_corresponding":false,"raw_author_name":"Hala Abdelhameed","raw_affiliation_strings":["Faculty of Computer and Artificial Intelligence, Fayoum University, Fayoum, Egypt","Khaybar Applied College, Taibah University, Medina, Saudi Arabia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Computer and Artificial Intelligence, Fayoum University, Fayoum, Egypt","institution_ids":["https://openalex.org/I66513531"]},{"raw_affiliation_string":"Khaybar Applied College, Taibah University, Medina, Saudi Arabia","institution_ids":["https://openalex.org/I23075662"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5041891989"],"corresponding_institution_ids":["https://openalex.org/I106778892"],"apc_list":{"value":1395,"currency":"USD","value_usd":1395},"apc_paid":{"value":1395,"currency":"USD","value_usd":1395},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.38079559,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"12","issue":null,"first_page":"e3629","last_page":"e3629"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9797999858856201,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9797999858856201,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.010099999606609344,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6596999764442444},{"id":"https://openalex.org/keywords/causality","display_name":"Causality (physics)","score":0.6186000108718872},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6047000288963318},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.5523999929428101},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.529699981212616},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.5281999707221985},{"id":"https://openalex.org/keywords/causal-inference","display_name":"Causal inference","score":0.41339999437332153}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6898000240325928},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6596999764442444},{"id":"https://openalex.org/C64357122","wikidata":"https://www.wikidata.org/wiki/Q1149766","display_name":"Causality (physics)","level":2,"score":0.6186000108718872},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6047000288963318},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.5523999929428101},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.529699981212616},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.5281999707221985},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49540001153945923},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.45719999074935913},{"id":"https://openalex.org/C158600405","wikidata":"https://www.wikidata.org/wiki/Q5054566","display_name":"Causal inference","level":2,"score":0.41339999437332153},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.4009999930858612},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3995000123977661},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3546000123023987},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.31790000200271606},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.30889999866485596},{"id":"https://openalex.org/C11671645","wikidata":"https://www.wikidata.org/wiki/Q5054567","display_name":"Causal model","level":2,"score":0.3028999865055084},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.2897000014781952},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.28610000014305115},{"id":"https://openalex.org/C6438553","wikidata":"https://www.wikidata.org/wiki/Q1185804","display_name":"Affective computing","level":2,"score":0.28029999136924744},{"id":"https://openalex.org/C206310091","wikidata":"https://www.wikidata.org/wiki/Q750859","display_name":"Emotion classification","level":2,"score":0.2612999975681305},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.26010000705718994}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.7717/peerj-cs.3629","is_oa":true,"landing_page_url":"https://doi.org/10.7717/peerj-cs.3629","pdf_url":null,"source":{"id":"https://openalex.org/S4210178049","display_name":"PeerJ Computer Science","issn_l":"2376-5992","issn":["2376-5992"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320104","host_organization_name":"PeerJ, Inc.","host_organization_lineage":["https://openalex.org/P4310320104"],"host_organization_lineage_names":["PeerJ, Inc."],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PeerJ Computer Science","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:928856e950d34dd89d2d37367a865072","is_oa":true,"landing_page_url":"https://doaj.org/article/928856e950d34dd89d2d37367a865072","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"PeerJ Computer Science, Vol 12, p e3629 (2026)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.7717/peerj-cs.3629","is_oa":true,"landing_page_url":"https://doi.org/10.7717/peerj-cs.3629","pdf_url":null,"source":{"id":"https://openalex.org/S4210178049","display_name":"PeerJ Computer Science","issn_l":"2376-5992","issn":["2376-5992"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320104","host_organization_name":"PeerJ, Inc.","host_organization_lineage":["https://openalex.org/P4310320104"],"host_organization_lineage_names":["PeerJ, Inc."],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PeerJ Computer Science","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.43176621198654175,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":46,"referenced_works":["https://openalex.org/W2055433631","https://openalex.org/W2132555391","https://openalex.org/W3044662450","https://openalex.org/W3155551469","https://openalex.org/W3204746395","https://openalex.org/W4200431468","https://openalex.org/W4206994745","https://openalex.org/W4225407275","https://openalex.org/W4289262425","https://openalex.org/W4295957212","https://openalex.org/W4306696332","https://openalex.org/W4308630605","https://openalex.org/W4308784450","https://openalex.org/W4312773554","https://openalex.org/W4313294355","https://openalex.org/W4322775637","https://openalex.org/W4367048669","https://openalex.org/W4376288601","https://openalex.org/W4381166482","https://openalex.org/W4381786694","https://openalex.org/W4385257056","https://openalex.org/W4385781891","https://openalex.org/W4386410420","https://openalex.org/W4386572391","https://openalex.org/W4388039626","https://openalex.org/W4388572082","https://openalex.org/W4389459413","https://openalex.org/W4389664932","https://openalex.org/W4390480791","https://openalex.org/W4391020222","https://openalex.org/W4392562716","https://openalex.org/W4394823827","https://openalex.org/W4396495439","https://openalex.org/W4399146389","https://openalex.org/W4399319756","https://openalex.org/W4399731136","https://openalex.org/W4399984819","https://openalex.org/W4402111545","https://openalex.org/W4402112107","https://openalex.org/W4402112295","https://openalex.org/W4405012805","https://openalex.org/W4406032612","https://openalex.org/W4406915043","https://openalex.org/W4409660542","https://openalex.org/W4410412777","https://openalex.org/W4412567193"],"related_works":[],"abstract_inverted_index":{"Emotion":[0],"recognition":[1,46,86],"plays":[2],"an":[3,44,63,89],"important":[4],"role":[5],"in":[6,19,26,153],"a":[7,28,56,95,118],"wide":[8],"range":[9],"of":[10,31,78,126],"application":[11],"domains.":[12],"Although":[13],"previous":[14],"studies":[15],"have":[16],"made":[17],"progress":[18],"this":[20],"domain,":[21],"they":[22],"often":[23],"fall":[24],"short":[25],"achieving":[27],"better":[29],"understanding":[30],"emotions":[32],"and":[33,52,105,133,145],"inferring":[34],"their":[35],"underlying":[36],"causes.":[37],"To":[38],"address":[39],"these":[40],"limitations,":[41],"we":[42],"propose":[43],"emotion":[45,90,120],"framework":[47,61],"that":[48,148],"integrates":[49,62],"visual,":[50],"audio,":[51],"textual":[53],"modalities":[54],"within":[55],"unified":[57],"architecture.":[58],"The":[59,136],"proposed":[60],"adaptive":[64],"cross-modal":[65],"attention":[66],"module":[67,73,93,122],"to":[68,101,156],"capture":[69],"inter-modal":[70],"interactions.":[71],"This":[72,108],"dynamically":[74],"adjusts":[75],"the":[76,114],"contribution":[77],"each":[79],"modality":[80],"based":[81],"on":[82,139],"contextual":[83],"relevance,":[84],"enhancing":[85],"accuracy.":[87],"Additionally,":[88],"causality":[91],"inference":[92],"uses":[94],"fine-tuned,":[96],"trainable":[97],"LLaMA2-Chat":[98],"(7B)":[99],"model":[100],"jointly":[102],"process":[103],"image":[104],"text":[106],"data.":[107],"identifies":[109],"word":[110],"clues":[111],"associated":[112],"with":[113],"expressed":[115],"emotions.":[116],"Furthermore,":[117],"real-time":[119],"feedback":[121],"delivers":[123],"instantaneous":[124],"assessments":[125],"emotional":[127],"states":[128],"during":[129],"conversations,":[130],"supporting":[131],"timely":[132],"context-aware":[134],"interventions.":[135],"experimental":[137],"results":[138],"four":[140],"datasets,":[141],"SEMAINE,":[142],"AESI,":[143],"ECF,":[144],"MER-2024,":[146],"demonstrate":[147],"our":[149],"method":[150],"achieves":[151],"improvements":[152],"F1-scores":[154],"compared":[155],"baselines.":[157]},"counts_by_year":[],"updated_date":"2026-05-06T08:25:59.206177","created_date":"2026-03-04T00:00:00"}