{"id":"https://openalex.org/W4416309285","doi":"https://doi.org/10.1186/s40537-025-01300-9","title":"A multimodal fusion model for real-time environment emotion recognition using audio-visual-textual features","display_name":"A multimodal fusion model for real-time environment emotion recognition using audio-visual-textual features","publication_year":2025,"publication_date":"2025-11-18","ids":{"openalex":"https://openalex.org/W4416309285","doi":"https://doi.org/10.1186/s40537-025-01300-9"},"language":"en","primary_location":{"id":"doi:10.1186/s40537-025-01300-9","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s40537-025-01300-9","pdf_url":"https://journalofbigdata.springeropen.com/counter/pdf/10.1186/s40537-025-01300-9","source":{"id":"https://openalex.org/S2737955091","display_name":"Journal Of Big Data","issn_l":"2196-1115","issn":["2196-1115"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Big Data","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://journalofbigdata.springeropen.com/counter/pdf/10.1186/s40537-025-01300-9","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5022985652","display_name":"Chhaya Gupta","orcid":null},"institutions":[{"id":"https://openalex.org/I4210131107","display_name":"Vivekananda Institute of Medical Sciences","ror":"https://ror.org/02j4gdg91","country_code":"IN","type":"education","lineage":["https://openalex.org/I4210131107"]},{"id":"https://openalex.org/I905734797","display_name":"Maharshi Dayanand University","ror":"https://ror.org/03kaab451","country_code":"IN","type":"education","lineage":["https://openalex.org/I905734797"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"Chhaya Gupta","raw_affiliation_strings":["Department of Computer Science and Applications, Maharshi Dayanand University, Rohtak, India","Faculty at School of Information Technology, Vivekananda Institute of Professional Studies-Technical Campus, Delhi, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Applications, Maharshi Dayanand University, Rohtak, India","institution_ids":["https://openalex.org/I905734797"]},{"raw_affiliation_string":"Faculty at School of Information Technology, Vivekananda Institute of Professional Studies-Technical Campus, Delhi, India","institution_ids":["https://openalex.org/I4210131107"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019909434","display_name":"Nasib Singh Gill","orcid":"https://orcid.org/0000-0002-8594-4320"},"institutions":[{"id":"https://openalex.org/I905734797","display_name":"Maharshi Dayanand University","ror":"https://ror.org/03kaab451","country_code":"IN","type":"education","lineage":["https://openalex.org/I905734797"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Nasib Singh Gill","raw_affiliation_strings":["Department of Computer Science and Applications, Maharshi Dayanand University, Rohtak, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Applications, Maharshi Dayanand University, Rohtak, India","institution_ids":["https://openalex.org/I905734797"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050940035","display_name":"Preeti Gulia","orcid":"https://orcid.org/0000-0001-8535-4016"},"institutions":[{"id":"https://openalex.org/I905734797","display_name":"Maharshi Dayanand University","ror":"https://ror.org/03kaab451","country_code":"IN","type":"education","lineage":["https://openalex.org/I905734797"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Preeti Gulia","raw_affiliation_strings":["Department of Computer Science and Applications, Maharshi Dayanand University, Rohtak, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Applications, Maharshi Dayanand University, Rohtak, India","institution_ids":["https://openalex.org/I905734797"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101528112","display_name":"Abhinav Kumar","orcid":"https://orcid.org/0000-0002-8091-3082"},"institutions":[{"id":"https://openalex.org/I3131779051","display_name":"Iraqi University","ror":"https://ror.org/01wfhkb67","country_code":"IQ","type":"education","lineage":["https://openalex.org/I3131779051"]},{"id":"https://openalex.org/I74319210","display_name":"Chitkara University","ror":"https://ror.org/057d6z539","country_code":"IN","type":"education","lineage":["https://openalex.org/I74319210"]}],"countries":["IN","IQ"],"is_corresponding":false,"raw_author_name":"Abhinav Kumar","raw_affiliation_strings":["Centre for Research Impact & Outcome, Chitkara University Institute of Engineering and Technology, Chitkara University, Rajpura, 140401, Punjab, India","Department of Mechanical Engineering and Renewable Energy, Technical Engineering College, The Islamic University, Najaf, Iraq"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Centre for Research Impact & Outcome, Chitkara University Institute of Engineering and Technology, Chitkara University, Rajpura, 140401, Punjab, India","institution_ids":["https://openalex.org/I74319210"]},{"raw_affiliation_string":"Department of Mechanical Engineering and Renewable Energy, Technical Engineering College, The Islamic University, Najaf, Iraq","institution_ids":["https://openalex.org/I3131779051"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057488732","display_name":"Hanen Karamti","orcid":"https://orcid.org/0000-0001-5162-2692"},"institutions":[{"id":"https://openalex.org/I106778892","display_name":"Princess Nourah bint Abdulrahman University","ror":"https://ror.org/05b0cyh02","country_code":"SA","type":"education","lineage":["https://openalex.org/I106778892"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Hanen Karamti","raw_affiliation_strings":["Department of Computer Sciences, College of Computer and Information Sciences, Princess Nourah bint Abdulrahman University, P.O. Box 84428, Riyadh, 11671, Saudi Arabia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Sciences, College of Computer and Information Sciences, Princess Nourah bint Abdulrahman University, P.O. Box 84428, Riyadh, 11671, Saudi Arabia","institution_ids":["https://openalex.org/I106778892"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038754189","display_name":"Demmelash Mollalign Moges","orcid":"https://orcid.org/0000-0003-3987-4525"},"institutions":[{"id":"https://openalex.org/I193649603","display_name":"Hawassa University","ror":"https://ror.org/04r15fz20","country_code":"ET","type":"education","lineage":["https://openalex.org/I193649603"]}],"countries":["ET"],"is_corresponding":false,"raw_author_name":"Demmelash Mollalign Moges","raw_affiliation_strings":["Department of Mathematics, Hawassa University, P.O. Box 05, Hawassa, Ethiopia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Mathematics, Hawassa University, P.O. Box 05, Hawassa, Ethiopia","institution_ids":["https://openalex.org/I193649603"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5116490187","display_name":"Imen Safra","orcid":null},"institutions":[{"id":"https://openalex.org/I106778892","display_name":"Princess Nourah bint Abdulrahman University","ror":"https://ror.org/05b0cyh02","country_code":"SA","type":"education","lineage":["https://openalex.org/I106778892"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Imen Safra","raw_affiliation_strings":["Department of Industrial and Systems Engineering, College of Engineering, Princess Nourah bint Abdulrahman University, P.O. Box 84428, Riyadh, 11671, Saudi Arabia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Industrial and Systems Engineering, College of Engineering, Princess Nourah bint Abdulrahman University, P.O. Box 84428, Riyadh, 11671, Saudi Arabia","institution_ids":["https://openalex.org/I106778892"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5022985652"],"corresponding_institution_ids":["https://openalex.org/I4210131107","https://openalex.org/I905734797"],"apc_list":{"value":1060,"currency":"GBP","value_usd":1300},"apc_paid":{"value":1060,"currency":"GBP","value_usd":1300},"fwci":7.2115,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.97174284,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":"12","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9557999968528748,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9557999968528748,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.002300000051036477,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.0017999999690800905,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.6610000133514404},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.6062999963760376},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.508400022983551},{"id":"https://openalex.org/keywords/object","display_name":"Object (grammar)","score":0.44040000438690186},{"id":"https://openalex.org/keywords/facial-expression","display_name":"Facial expression","score":0.41609999537467957},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.4124999940395355},{"id":"https://openalex.org/keywords/emotion-classification","display_name":"Emotion classification","score":0.3831000030040741},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.3790000081062317},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.3790000081062317}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8485999703407288},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6761999726295471},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.6610000133514404},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.6062999963760376},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.508400022983551},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.44040000438690186},{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.41609999537467957},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.4124999940395355},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.38580000400543213},{"id":"https://openalex.org/C206310091","wikidata":"https://www.wikidata.org/wiki/Q750859","display_name":"Emotion classification","level":2,"score":0.3831000030040741},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.3790000081062317},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3790000081062317},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.376800000667572},{"id":"https://openalex.org/C66905080","wikidata":"https://www.wikidata.org/wiki/Q17005494","display_name":"Binary classification","level":3,"score":0.3531999886035919},{"id":"https://openalex.org/C42747912","wikidata":"https://www.wikidata.org/wiki/Q1048447","display_name":"Multiplicative function","level":2,"score":0.34709998965263367},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.3467000126838684},{"id":"https://openalex.org/C64876066","wikidata":"https://www.wikidata.org/wiki/Q5141226","display_name":"Cognitive neuroscience of visual object recognition","level":3,"score":0.3368000090122223},{"id":"https://openalex.org/C121687571","wikidata":"https://www.wikidata.org/wiki/Q4677630","display_name":"Activity recognition","level":2,"score":0.32409998774528503},{"id":"https://openalex.org/C6438553","wikidata":"https://www.wikidata.org/wiki/Q1185804","display_name":"Affective computing","level":2,"score":0.3190000057220459},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.31610000133514404},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.30079999566078186},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3003000020980835},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.29910001158714294},{"id":"https://openalex.org/C2988148770","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion detection","level":3,"score":0.2987000048160553},{"id":"https://openalex.org/C2776151529","wikidata":"https://www.wikidata.org/wiki/Q3045304","display_name":"Object detection","level":3,"score":0.26249998807907104},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.26109999418258667},{"id":"https://openalex.org/C87335442","wikidata":"https://www.wikidata.org/wiki/Q2494345","display_name":"Local binary patterns","level":4,"score":0.25369998812675476}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1186/s40537-025-01300-9","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s40537-025-01300-9","pdf_url":"https://journalofbigdata.springeropen.com/counter/pdf/10.1186/s40537-025-01300-9","source":{"id":"https://openalex.org/S2737955091","display_name":"Journal Of Big Data","issn_l":"2196-1115","issn":["2196-1115"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Big Data","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:0f60b98b58d04a0ab89da5351e0fe568","is_oa":true,"landing_page_url":"https://doaj.org/article/0f60b98b58d04a0ab89da5351e0fe568","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of Big Data, Vol 12, Iss 1, Pp 1-29 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1186/s40537-025-01300-9","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s40537-025-01300-9","pdf_url":"https://journalofbigdata.springeropen.com/counter/pdf/10.1186/s40537-025-01300-9","source":{"id":"https://openalex.org/S2737955091","display_name":"Journal Of Big Data","issn_l":"2196-1115","issn":["2196-1115"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Big Data","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3783886058","display_name":null,"funder_award_id":"PNURSP2025R817","funder_id":"https://openalex.org/F4320322484","funder_display_name":"Princess Nourah Bint Abdulrahman University"}],"funders":[{"id":"https://openalex.org/F4320322484","display_name":"Princess Nourah Bint Abdulrahman University","ror":"https://ror.org/05b0cyh02"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416309285.pdf"},"referenced_works_count":33,"referenced_works":["https://openalex.org/W2406223855","https://openalex.org/W2883409523","https://openalex.org/W2948242564","https://openalex.org/W2964216663","https://openalex.org/W2996906606","https://openalex.org/W3013022628","https://openalex.org/W3094524767","https://openalex.org/W3113497772","https://openalex.org/W3195903076","https://openalex.org/W3213879871","https://openalex.org/W4210940289","https://openalex.org/W4221154966","https://openalex.org/W4282598382","https://openalex.org/W4289874551","https://openalex.org/W4292411680","https://openalex.org/W4297499129","https://openalex.org/W4306167427","https://openalex.org/W4306250392","https://openalex.org/W4306955484","https://openalex.org/W4313542967","https://openalex.org/W4313646621","https://openalex.org/W4320002631","https://openalex.org/W4321022102","https://openalex.org/W4362496488","https://openalex.org/W4363677626","https://openalex.org/W4377292302","https://openalex.org/W4386076442","https://openalex.org/W4388819630","https://openalex.org/W4393969642","https://openalex.org/W4410118057","https://openalex.org/W4411358758","https://openalex.org/W4411504396","https://openalex.org/W4413119630"],"related_works":[],"abstract_inverted_index":{"Multimodal":[0],"combines":[1],"multiple":[2],"modalities":[3],"to":[4,9,20,144],"create":[5],"insightful":[6],"conclusions":[7],"or":[8],"make":[10],"more":[11],"precise":[12],"predictions.":[13],"Nowadays,":[14],"the":[15,46,54,61,73,89,117,136,149,164,168],"multimodal":[16,37],"concept":[17],"is":[18,102,141,178,185],"used":[19,128,143,160],"identify":[21],"human":[22,40,96],"emotions":[23],"precisely.":[24],"This":[25],"study":[26],"proposes":[27],"a":[28,64,123],"three-stage":[29],"novel":[30],"M-fusHER":[31,171],"(Multimodal":[32],"fusion":[33],"Human":[34],"Emotion":[35],"Recognition)":[36],"model":[38,126,170],"for":[39,95,129,161],"emotion":[41,97],"recognition":[42,98],"in":[43,85],"real-time":[44,92],"with":[45,60,69],"help":[47,62],"of":[48,63,167],"text,":[49,80,112],"audio,":[50,111],"and":[51,77,81,113,133,146,157,163,175,181],"videos.":[52],"In":[53,72,88],"first":[55],"stage,":[56,75,91],"features":[57,132],"are":[58,83,107,159],"extracted":[59],"convolutional":[65],"neural":[66],"network":[67],"merged":[68],"multiplicative":[70,139],"LSTM.":[71],"second":[74],"video":[76],"audio":[78,82],"data,":[79],"fused":[84],"binary":[86],"form.":[87],"third":[90],"object":[93,121],"detection":[94,165],"on":[99,172],"real":[100],"videos":[101,114],"implemented.":[103],"The":[104,138],"experimental":[105],"results":[106],"obtained":[108],"by":[109,115],"fusing":[110],"considering":[116],"standard":[118],"features.":[119,151],"For":[120],"detection,":[122],"fine-tuned":[124],"YOLOv6":[125],"was":[127],"detecting":[130],"facial":[131],"expressions":[134],"from":[135,148],"video.":[137],"LSTM":[140],"also":[142],"extract":[145],"learn":[147],"text":[150],"Three":[152],"datasets,":[153],"i.e.,":[154],"IEMOCAP,":[155,173],"MOSEI,":[156,174],"MELD":[158,176],"implementation,":[162],"accuracy":[166],"proposed":[169],"datasets":[177],"95.45%,":[179],"88.76%,":[180],"95.41%":[182],"approximately,":[183],"which":[184],"quite":[186],"encouraging.":[187]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2}],"updated_date":"2026-05-01T08:36:08.643496","created_date":"2025-11-18T00:00:00"}