{"id":"https://openalex.org/W7132821851","doi":"https://doi.org/10.1007/s44443-026-00610-z","title":"FMII: a unified intra- and inter-modal interaction framework for robust audio-visual emotion recognition","display_name":"FMII: a unified intra- and inter-modal interaction framework for robust audio-visual emotion recognition","publication_year":2026,"publication_date":"2026-02-28","ids":{"openalex":"https://openalex.org/W7132821851","doi":"https://doi.org/10.1007/s44443-026-00610-z"},"language":"en","primary_location":{"id":"doi:10.1007/s44443-026-00610-z","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s44443-026-00610-z","pdf_url":"https://link.springer.com/content/pdf/10.1007/s44443-026-00610-z.pdf","source":{"id":"https://openalex.org/S2764955546","display_name":"Journal of King Saud University - Computer and Information Sciences","issn_l":"1319-1578","issn":["1319-1578","2213-1248"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of King Saud University Computer and Information Sciences","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://link.springer.com/content/pdf/10.1007/s44443-026-00610-z.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030948239","display_name":"Zijian Sun","orcid":null},"institutions":[{"id":"https://openalex.org/I39333907","display_name":"Yanshan University","ror":"https://ror.org/02txfnf15","country_code":"CN","type":"education","lineage":["https://openalex.org/I39333907"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zijian Sun","raw_affiliation_strings":["School of Information Science and Engineering, Yanshan University, Qinhuangdao, China","The Key Laboratory for Special Fiber and Fiber Sensor of Hebei Province, Yanshan University, Qinhuangdao, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Information Science and Engineering, Yanshan University, Qinhuangdao, China","institution_ids":["https://openalex.org/I39333907"]},{"raw_affiliation_string":"The Key Laboratory for Special Fiber and Fiber Sensor of Hebei Province, Yanshan University, Qinhuangdao, China","institution_ids":["https://openalex.org/I39333907"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127628342","display_name":"Haoran Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I39333907","display_name":"Yanshan University","ror":"https://ror.org/02txfnf15","country_code":"CN","type":"education","lineage":["https://openalex.org/I39333907"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Haoran Liu","raw_affiliation_strings":["School of Information Science and Engineering, Yanshan University, Qinhuangdao, China","The Key Laboratory for Special Fiber and Fiber Sensor of Hebei Province, Yanshan University, Qinhuangdao, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Information Science and Engineering, Yanshan University, Qinhuangdao, China","institution_ids":["https://openalex.org/I39333907"]},{"raw_affiliation_string":"The Key Laboratory for Special Fiber and Fiber Sensor of Hebei Province, Yanshan University, Qinhuangdao, China","institution_ids":["https://openalex.org/I39333907"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100697598","display_name":"Yaqian Li","orcid":"https://orcid.org/0000-0003-1032-9910"},"institutions":[{"id":"https://openalex.org/I39333907","display_name":"Yanshan University","ror":"https://ror.org/02txfnf15","country_code":"CN","type":"education","lineage":["https://openalex.org/I39333907"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaqian Li","raw_affiliation_strings":["Hebei Key Laboratory of Industrial Computer Control Engineering, Yanshan University, Qinhuangdao, China","School of Electrical Engineering, Yanshan University, Qinhuangdao, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hebei Key Laboratory of Industrial Computer Control Engineering, Yanshan University, Qinhuangdao, China","institution_ids":["https://openalex.org/I39333907"]},{"raw_affiliation_string":"School of Electrical Engineering, Yanshan University, Qinhuangdao, China","institution_ids":["https://openalex.org/I39333907"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127598370","display_name":"Haibin Li","orcid":null},"institutions":[{"id":"https://openalex.org/I39333907","display_name":"Yanshan University","ror":"https://ror.org/02txfnf15","country_code":"CN","type":"education","lineage":["https://openalex.org/I39333907"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haibin Li","raw_affiliation_strings":["Hebei Key Laboratory of Industrial Computer Control Engineering, Yanshan University, Qinhuangdao, China","School of Electrical Engineering, Yanshan University, Qinhuangdao, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hebei Key Laboratory of Industrial Computer Control Engineering, Yanshan University, Qinhuangdao, China","institution_ids":["https://openalex.org/I39333907"]},{"raw_affiliation_string":"School of Electrical Engineering, Yanshan University, Qinhuangdao, China","institution_ids":["https://openalex.org/I39333907"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127584801","display_name":"Wenming Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I39333907","display_name":"Yanshan University","ror":"https://ror.org/02txfnf15","country_code":"CN","type":"education","lineage":["https://openalex.org/I39333907"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenming Zhang","raw_affiliation_strings":["Hebei Key Laboratory of Industrial Computer Control Engineering, Yanshan University, Qinhuangdao, China","School of Electrical Engineering, Yanshan University, Qinhuangdao, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hebei Key Laboratory of Industrial Computer Control Engineering, Yanshan University, Qinhuangdao, China","institution_ids":["https://openalex.org/I39333907"]},{"raw_affiliation_string":"School of Electrical Engineering, Yanshan University, Qinhuangdao, China","institution_ids":["https://openalex.org/I39333907"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5127695171","display_name":"Tao Song","orcid":null},"institutions":[{"id":"https://openalex.org/I39333907","display_name":"Yanshan University","ror":"https://ror.org/02txfnf15","country_code":"CN","type":"education","lineage":["https://openalex.org/I39333907"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tao Song","raw_affiliation_strings":["Hebei Key Laboratory of Industrial Computer Control Engineering, Yanshan University, Qinhuangdao, China","School of Electrical Engineering, Yanshan University, Qinhuangdao, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hebei Key Laboratory of Industrial Computer Control Engineering, Yanshan University, Qinhuangdao, China","institution_ids":["https://openalex.org/I39333907"]},{"raw_affiliation_string":"School of Electrical Engineering, Yanshan University, Qinhuangdao, China","institution_ids":["https://openalex.org/I39333907"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5127628342"],"corresponding_institution_ids":["https://openalex.org/I39333907"],"apc_list":{"value":1350,"currency":"USD","value_usd":1350},"apc_paid":{"value":1350,"currency":"USD","value_usd":1350},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.2429429,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"38","issue":"5","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9574999809265137,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9574999809265137,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.004800000227987766,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.004600000102072954,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.3400000035762787},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.32030001282691956},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.29109999537467957},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.2863999903202057},{"id":"https://openalex.org/keywords/constraint","display_name":"Constraint (computer-aided design)","score":0.2766000032424927}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5698000192642212},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4788999855518341},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3400000035762787},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.32030001282691956},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.29109999537467957},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2863999903202057},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.2766000032424927},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.273499995470047},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.2680000066757202},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2648000121116638},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.258899986743927},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2526000142097473}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/s44443-026-00610-z","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s44443-026-00610-z","pdf_url":"https://link.springer.com/content/pdf/10.1007/s44443-026-00610-z.pdf","source":{"id":"https://openalex.org/S2764955546","display_name":"Journal of King Saud University - Computer and Information Sciences","issn_l":"1319-1578","issn":["1319-1578","2213-1248"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of King Saud University Computer and Information Sciences","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:2466d3125ecd4de8bd0243ef80a1a44f","is_oa":true,"landing_page_url":"https://doaj.org/article/2466d3125ecd4de8bd0243ef80a1a44f","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of King Saud University: Computer and Information Sciences, Vol 38, Iss 5 (2026)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1007/s44443-026-00610-z","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s44443-026-00610-z","pdf_url":"https://link.springer.com/content/pdf/10.1007/s44443-026-00610-z.pdf","source":{"id":"https://openalex.org/S2764955546","display_name":"Journal of King Saud University - Computer and Information Sciences","issn_l":"1319-1578","issn":["1319-1578","2213-1248"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of King Saud University Computer and Information Sciences","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3055821539","display_name":null,"funder_award_id":"D2024203008","funder_id":"https://openalex.org/F4320322163","funder_display_name":"Natural Science Foundation of Hebei Province"}],"funders":[{"id":"https://openalex.org/F4320322163","display_name":"Natural Science Foundation of Hebei Province","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7132821851.pdf","grobid_xml":"https://content.openalex.org/works/W7132821851.grobid-xml"},"referenced_works_count":70,"referenced_works":["https://openalex.org/W2030931454","https://openalex.org/W2126552487","https://openalex.org/W2803193013","https://openalex.org/W2883409523","https://openalex.org/W2962793908","https://openalex.org/W3093051361","https://openalex.org/W3094550259","https://openalex.org/W3113270693","https://openalex.org/W3174977508","https://openalex.org/W3186351311","https://openalex.org/W3214432797","https://openalex.org/W4210580780","https://openalex.org/W4220829848","https://openalex.org/W4221139382","https://openalex.org/W4223460604","https://openalex.org/W4224932526","https://openalex.org/W4225959162","https://openalex.org/W4285149123","https://openalex.org/W4286370724","https://openalex.org/W4304092664","https://openalex.org/W4308065851","https://openalex.org/W4312596733","https://openalex.org/W4312857458","https://openalex.org/W4322736091","https://openalex.org/W4323664393","https://openalex.org/W4327852215","https://openalex.org/W4361008338","https://openalex.org/W4372260527","https://openalex.org/W4386076642","https://openalex.org/W4389733467","https://openalex.org/W4390414889","https://openalex.org/W4390872007","https://openalex.org/W4390873518","https://openalex.org/W4391302483","https://openalex.org/W4391621171","https://openalex.org/W4392902910","https://openalex.org/W4393084501","https://openalex.org/W4393147302","https://openalex.org/W4393181080","https://openalex.org/W4393397422","https://openalex.org/W4400018045","https://openalex.org/W4400276083","https://openalex.org/W4400975545","https://openalex.org/W4402352780","https://openalex.org/W4402703093","https://openalex.org/W4402832821","https://openalex.org/W4402915969","https://openalex.org/W4402916570","https://openalex.org/W4403532823","https://openalex.org/W4403792017","https://openalex.org/W4403924124","https://openalex.org/W4404902039","https://openalex.org/W4404969221","https://openalex.org/W4405271539","https://openalex.org/W4406387540","https://openalex.org/W4406612302","https://openalex.org/W4407893594","https://openalex.org/W4408353970","https://openalex.org/W4408355899","https://openalex.org/W4408792776","https://openalex.org/W4409262900","https://openalex.org/W4409472566","https://openalex.org/W4410027386","https://openalex.org/W4410536426","https://openalex.org/W4410586919","https://openalex.org/W4410987222","https://openalex.org/W4413030505","https://openalex.org/W4414203465","https://openalex.org/W4414271621","https://openalex.org/W7131851128"],"related_works":[],"abstract_inverted_index":{"In":[0],"recent":[1],"years,":[2],"Audio-Visual":[3],"Emotion":[4],"Recognition":[5],"(AVER)":[6],"has":[7],"gained":[8],"increasing":[9],"attention":[10,121],"due":[11],"to":[12,91,145,193],"its":[13],"immense":[14],"potential":[15],"in":[16,34,111,149,207],"human-computer":[17],"interaction":[18],"and":[19,41,62,87,127,132,138,165,176,200,220],"multimedia":[20],"applications.":[21],"Despite":[22],"significant":[23],"progress,":[24],"effective":[25],"audio-visual":[26],"emotion":[27,210],"recognition":[28],"still":[29],"faces":[30],"challenges,":[31],"including":[32],"difficulty":[33],"synergizing":[35],"time-frequency":[36,73,85],"evidence,":[37],"weak":[38],"cross-modal":[39,120],"consistency":[40,110],"underutilization":[42],"of":[43,109,140],"complementary":[44,156,181,201],"cues,":[45,202],"as":[46,48],"well":[47],"redundancy-induced":[49],"interference.":[50],"To":[51],"address":[52,106],"these":[53],"issues,":[54],"we":[55,70,104,152,169],"propose":[56],"a":[57,72,116,154],"novel":[58],"framework":[59],"for":[60],"intra-modal":[61,68,196],"inter-modal":[63,102,198],"interactions":[64],"(FMII).":[65],"At":[66,100],"the":[67,101,107,136,141,230],"level,":[69,103],"design":[71],"adaptive":[74,98],"fusion":[75,122],"strategy,":[76],"which":[77],"enhances":[78],"audio":[79],"representations\u2019":[80],"robustness":[81,139],"by":[82,114],"learning":[83],"bidirectional":[84],"mappings":[86],"applying":[88],"soft":[89],"masks":[90],"filter":[92],"key":[93],"segments,":[94],"along":[95],"with":[96],"domain-level":[97],"re-scaling.":[99],"first":[105],"lack":[108],"modality-invariant":[112],"representations":[113],"developing":[115],"hierarchical":[117],"multi-granularity":[118],"dynamic":[119],"strategy.":[123,159],"This":[124],"strategy":[125],"captures":[126],"balances":[128],"global":[129],"semantic":[130],"cues":[131,182],"local":[133],"details,":[134],"improving":[135],"discriminability":[137],"fused":[142],"representation.":[143],"Additionally,":[144],"mitigate":[146],"redundant":[147,186],"interference":[148],"modality-specific":[150],"representations,":[151],"introduce":[153],"feature-level":[155],"information":[157,164],"disentanglement":[158],"By":[160],"maximizing":[161],"discriminative":[162],"mutual":[163],"reducing":[166],"cross-subspace":[167],"interference,":[168],"decompose":[170],"each":[171],"modality\u2019s":[172],"features":[173],"into":[174],"task-relevant":[175],"task-irrelevant":[177],"subspaces,":[178],"preserving":[179],"useful":[180],"while":[183,203],"filtering":[184],"out":[185],"information.":[187],"These":[188],"combined":[189],"strategies":[190],"enable":[191],"FMII":[192],"effectively":[194],"capture":[195],"information,":[197],"consistency,":[199],"minimizing":[204],"noise,":[205],"resulting":[206],"robust":[208],"unified":[209],"representations.":[211],"Comprehensive":[212],"experiments":[213],"on":[214,229],"three":[215],"popular":[216],"datasets":[217],"(RAVDESS,":[218],"CREMA-D,":[219],"CMU-MOSEI)":[221],"demonstrate":[222],"that":[223],"our":[224],"method":[225],"achieves":[226],"superior":[227],"performance":[228],"AVER":[231],"task.":[232]},"counts_by_year":[],"updated_date":"2026-06-20T20:08:15.867695","created_date":"2026-03-01T00:00:00"}