{"id":"https://openalex.org/W4413104916","doi":"https://doi.org/10.1109/jbhi.2025.3597398","title":"Multimodal Fusion of Behavioral and Physiological Signals for Enhanced Emotion Recognition via Feature Decoupling and Knowledge Transfer","display_name":"Multimodal Fusion of Behavioral and Physiological Signals for Enhanced Emotion Recognition via Feature Decoupling and Knowledge Transfer","publication_year":2025,"publication_date":"2025-08-11","ids":{"openalex":"https://openalex.org/W4413104916","doi":"https://doi.org/10.1109/jbhi.2025.3597398","pmid":"https://pubmed.ncbi.nlm.nih.gov/40788806"},"language":"en","primary_location":{"id":"doi:10.1109/jbhi.2025.3597398","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jbhi.2025.3597398","pdf_url":null,"source":{"id":"https://openalex.org/S2495854775","display_name":"IEEE Journal of Biomedical and Health Informatics","issn_l":"2168-2194","issn":["2168-2194","2168-2208"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Biomedical and Health Informatics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101828945","display_name":"Hongxiang Gao","orcid":"https://orcid.org/0000-0003-4121-0250"},"institutions":[{"id":"https://openalex.org/I4387154005","display_name":"State Key Laboratory of Digital Medical Engineering","ror":"https://ror.org/03ab0at74","country_code":null,"type":"facility","lineage":["https://openalex.org/I4387154005","https://openalex.org/I76569877"]},{"id":"https://openalex.org/I76569877","display_name":"Southeast University","ror":"https://ror.org/04ct4d772","country_code":"CN","type":"education","lineage":["https://openalex.org/I76569877"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Hongxiang Gao","raw_affiliation_strings":["State Key Laboratory of Digital Medical Engineering, School of Instrument Science and Engineering, Southeast University, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Digital Medical Engineering, School of Instrument Science and Engineering, Southeast University, Nanjing, China","institution_ids":["https://openalex.org/I76569877","https://openalex.org/I4387154005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100666505","display_name":"Zhipeng Cai","orcid":"https://orcid.org/0000-0003-3653-6838"},"institutions":[{"id":"https://openalex.org/I76569877","display_name":"Southeast University","ror":"https://ror.org/04ct4d772","country_code":"CN","type":"education","lineage":["https://openalex.org/I76569877"]},{"id":"https://openalex.org/I4387154005","display_name":"State Key Laboratory of Digital Medical Engineering","ror":"https://ror.org/03ab0at74","country_code":null,"type":"facility","lineage":["https://openalex.org/I4387154005","https://openalex.org/I76569877"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhipeng Cai","raw_affiliation_strings":["State Key Laboratory of Digital Medical Engineering, School of Instrument Science and Engineering, Southeast University, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Digital Medical Engineering, School of Instrument Science and Engineering, Southeast University, Nanjing, China","institution_ids":["https://openalex.org/I76569877","https://openalex.org/I4387154005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065691929","display_name":"Xingyao Wang","orcid":"https://orcid.org/0000-0002-3483-8624"},"institutions":[{"id":"https://openalex.org/I3004594783","display_name":"Institute of High Performance Computing","ror":"https://ror.org/02n0ejh50","country_code":"SG","type":"facility","lineage":["https://openalex.org/I115228651","https://openalex.org/I3004594783","https://openalex.org/I91275662"]},{"id":"https://openalex.org/I115228651","display_name":"Agency for Science, Technology and Research","ror":"https://ror.org/036wvzt09","country_code":"SG","type":"government","lineage":["https://openalex.org/I115228651"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Xingyao Wang","raw_affiliation_strings":["Institute for High Performance Computing, A*STAR, Singapore"],"affiliations":[{"raw_affiliation_string":"Institute for High Performance Computing, A*STAR, Singapore","institution_ids":["https://openalex.org/I3004594783","https://openalex.org/I115228651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100340964","display_name":"Min Wu","orcid":"https://orcid.org/0000-0003-0977-3600"},"institutions":[{"id":"https://openalex.org/I3005327000","display_name":"Institute for Infocomm Research","ror":"https://ror.org/053rfa017","country_code":"SG","type":"facility","lineage":["https://openalex.org/I115228651","https://openalex.org/I3005327000","https://openalex.org/I91275662"]},{"id":"https://openalex.org/I115228651","display_name":"Agency for Science, Technology and Research","ror":"https://ror.org/036wvzt09","country_code":"SG","type":"government","lineage":["https://openalex.org/I115228651"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Min Wu","raw_affiliation_strings":["Institute for Infocomm Research, A*STAR, Singapore"],"affiliations":[{"raw_affiliation_string":"Institute for Infocomm Research, A*STAR, Singapore","institution_ids":["https://openalex.org/I3005327000","https://openalex.org/I115228651"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100321186","display_name":"Chengyu Liu","orcid":"https://orcid.org/0000-0003-1965-3020"},"institutions":[{"id":"https://openalex.org/I76569877","display_name":"Southeast University","ror":"https://ror.org/04ct4d772","country_code":"CN","type":"education","lineage":["https://openalex.org/I76569877"]},{"id":"https://openalex.org/I4387154005","display_name":"State Key Laboratory of Digital Medical Engineering","ror":"https://ror.org/03ab0at74","country_code":null,"type":"facility","lineage":["https://openalex.org/I4387154005","https://openalex.org/I76569877"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chengyu Liu","raw_affiliation_strings":["State Key Laboratory of Digital Medical Engineering, School of Instrument Science and Engineering, Southeast University, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Digital Medical Engineering, School of Instrument Science and Engineering, Southeast University, Nanjing, China","institution_ids":["https://openalex.org/I76569877","https://openalex.org/I4387154005"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5101828945"],"corresponding_institution_ids":["https://openalex.org/I4387154005","https://openalex.org/I76569877"],"apc_list":null,"apc_paid":null,"fwci":6.6459,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.96747223,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"30","issue":"1","first_page":"208","last_page":"218"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9394000172615051,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9394000172615051,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6068486571311951},{"id":"https://openalex.org/keywords/decoupling","display_name":"Decoupling (probability)","score":0.5626954436302185},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.5393436551094055},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.5123285055160522},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5078919529914856},{"id":"https://openalex.org/keywords/affective-computing","display_name":"Affective computing","score":0.48083576560020447},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4358733892440796},{"id":"https://openalex.org/keywords/fusion","display_name":"Fusion","score":0.4259803891181946},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.4252339005470276},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.42266008257865906},{"id":"https://openalex.org/keywords/sensor-fusion","display_name":"Sensor fusion","score":0.4109598994255066},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.14072152972221375}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6068486571311951},{"id":"https://openalex.org/C205606062","wikidata":"https://www.wikidata.org/wiki/Q5249645","display_name":"Decoupling (probability)","level":2,"score":0.5626954436302185},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.5393436551094055},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5123285055160522},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5078919529914856},{"id":"https://openalex.org/C6438553","wikidata":"https://www.wikidata.org/wiki/Q1185804","display_name":"Affective computing","level":2,"score":0.48083576560020447},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4358733892440796},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.4259803891181946},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.4252339005470276},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.42266008257865906},{"id":"https://openalex.org/C33954974","wikidata":"https://www.wikidata.org/wiki/Q486494","display_name":"Sensor fusion","level":2,"score":0.4109598994255066},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.14072152972221375},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C133731056","wikidata":"https://www.wikidata.org/wiki/Q4917288","display_name":"Control engineering","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D004644","descriptor_name":"Emotions","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":true},{"descriptor_ui":"D004644","descriptor_name":"Emotions","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":true},{"descriptor_ui":"D004644","descriptor_name":"Emotions","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":true},{"descriptor_ui":"D004644","descriptor_name":"Emotions","qualifier_ui":"Q000502","qualifier_name":"physiology","is_major_topic":true},{"descriptor_ui":"D004644","descriptor_name":"Emotions","qualifier_ui":"Q000502","qualifier_name":"physiology","is_major_topic":true},{"descriptor_ui":"D004644","descriptor_name":"Emotions","qualifier_ui":"Q000502","qualifier_name":"physiology","is_major_topic":true},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012815","descriptor_name":"Signal Processing, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012815","descriptor_name":"Signal Processing, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012815","descriptor_name":"Signal Processing, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":2,"locations":[{"id":"doi:10.1109/jbhi.2025.3597398","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jbhi.2025.3597398","pdf_url":null,"source":{"id":"https://openalex.org/S2495854775","display_name":"IEEE Journal of Biomedical and Health Informatics","issn_l":"2168-2194","issn":["2168-2194","2168-2208"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Biomedical and Health Informatics","raw_type":"journal-article"},{"id":"pmid:40788806","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40788806","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE journal of biomedical and health informatics","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W1985541164","https://openalex.org/W2002055708","https://openalex.org/W2029334490","https://openalex.org/W2045561515","https://openalex.org/W2087347434","https://openalex.org/W2122098299","https://openalex.org/W2129408021","https://openalex.org/W2133085034","https://openalex.org/W2278113816","https://openalex.org/W2619383789","https://openalex.org/W2766736793","https://openalex.org/W2767618761","https://openalex.org/W2786768213","https://openalex.org/W2790404832","https://openalex.org/W2883630736","https://openalex.org/W2903959724","https://openalex.org/W2964051877","https://openalex.org/W3024961463","https://openalex.org/W3093051361","https://openalex.org/W3093125198","https://openalex.org/W3094378471","https://openalex.org/W3205519684","https://openalex.org/W3206241967","https://openalex.org/W4210611093","https://openalex.org/W4223499953","https://openalex.org/W4225162736","https://openalex.org/W4281710448","https://openalex.org/W4297510826","https://openalex.org/W4304092664","https://openalex.org/W4310064432","https://openalex.org/W4360595168","https://openalex.org/W4367281761","https://openalex.org/W4386075879","https://openalex.org/W4386757131","https://openalex.org/W4399849687"],"related_works":["https://openalex.org/W2132659060","https://openalex.org/W2031992971","https://openalex.org/W3214791684","https://openalex.org/W2152662039","https://openalex.org/W3080495370","https://openalex.org/W4285597148","https://openalex.org/W2901531394","https://openalex.org/W1559262936","https://openalex.org/W2134707158","https://openalex.org/W4321599321"],"abstract_inverted_index":{"Multimodal":[0,43],"emotion":[1,180],"recognition":[2],"has":[3],"emerged":[4],"as":[5],"a":[6,41,61,68,88,172],"promising":[7],"direction":[8],"for":[9,178,186],"capturing":[10],"the":[11,98,149,158,184],"complexity":[12],"of":[13,148,160],"human":[14],"affective":[15,187],"states":[16],"by":[17,140],"integrating":[18],"physiological":[19],"and":[20,32,52,55,76,90,115,119,124,153,164,175,182],"behavioral":[21],"signals.":[22],"However,":[23],"challenges":[24],"remain":[25],"in":[26,189],"addressing":[27],"feature":[28,162],"redundancy,":[29],"modality":[30,92,104,151],"heterogeneity,":[31],"insufficient":[33],"inter-modal":[34],"supervision.":[35],"In":[36,144],"this":[37],"paper,":[38],"we":[39,66,86],"propose":[40],"novel":[42],"Disentangled":[44],"Knowledge":[45],"Distillation":[46],"framework":[47,177],"that":[48,73,130],"explicitly":[49],"disentangles":[50],"modality-shared":[51],"modality-specific":[53],"features":[54],"enhances":[56],"cross-modal":[57],"knowledge":[58,166],"transfer":[59,100,167],"via":[60],"graph-based":[62],"distillation":[63],"module.":[64],"Specifically,":[65],"introduce":[67],"dual-stream":[69],"representation":[70],"learning":[71],"architecture":[72],"separates":[74],"common":[75],"unique":[77],"subspaces":[78],"across":[79],"modalities.":[80],"To":[81],"facilitate":[82],"effective":[83],"information":[84],"interaction,":[85],"design":[87],"directed":[89],"learnable":[91],"graph,":[93],"where":[94],"each":[95],"edge":[96],"represents":[97],"semantic":[99],"strength":[101],"from":[102],"one":[103],"to":[105],"another.":[106],"We":[107],"validate":[108],"our":[109,131,161],"method":[110,132],"on":[111],"two":[112],"benchmark":[113],"datasets-MAHNOB-HCI":[114],"DEAP-for":[116],"both":[117],"regression":[118],"classification":[120],"tasks,":[121],"under":[122],"subject-dependent":[123],"subject-independent":[125],"protocols.":[126],"Experimental":[127],"results":[128],"demonstrate":[129],"achieves":[133],"state-of-the-art":[134],"performance,":[135],"with":[136],"statistical":[137],"significance":[138],"confirmed":[139],"paired":[141],"two-tailed":[142],"$t$-tests.":[143],"addition,":[145],"qualitative":[146],"analysis":[147],"learned":[150],"graph":[152],"t-SNE":[154],"embeddings":[155],"further":[156],"illustrates":[157],"effectiveness":[159],"disentanglement":[163],"dynamic":[165],"design.":[168],"This":[169],"work":[170],"offers":[171],"unified,":[173],"interpretable,":[174],"robust":[176],"multimodal":[179],"understanding":[181],"lays":[183],"foundation":[185],"computing":[188],"real-world":[190],"human-machine":[191],"interaction":[192],"scenarios.":[193]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-01-10T23:39:48.068659","created_date":"2025-10-10T00:00:00"}