{"id":"https://openalex.org/W7163546700","doi":"https://doi.org/10.1016/j.inffus.2026.104530","title":"Hinting the unknown: Effective open-set multimodal emotion recognition with a hierarchical cross-modal emotion-interactive prompting approach","display_name":"Hinting the unknown: Effective open-set multimodal emotion recognition with a hierarchical cross-modal emotion-interactive prompting approach","publication_year":2026,"publication_date":"2026-06-04","ids":{"openalex":"https://openalex.org/W7163546700","doi":"https://doi.org/10.1016/j.inffus.2026.104530"},"language":"en","primary_location":{"id":"doi:10.1016/j.inffus.2026.104530","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.inffus.2026.104530","pdf_url":null,"source":{"id":"https://openalex.org/S7560371","display_name":"Information Fusion","issn_l":"1566-2535","issn":["1566-2535","1872-6305"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Fusion","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.inffus.2026.104530","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137905914","display_name":"Yuanyuan Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I3124059619","display_name":"China University of Geosciences","ror":"https://ror.org/04gcegc37","country_code":"CN","type":"education","lineage":["https://openalex.org/I3124059619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuanyuan Liu","raw_affiliation_strings":["School of Computer Science, China University of Geosciences (Wuhan), Wuhan, Hubei, 430074, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, China University of Geosciences (Wuhan), Wuhan, Hubei, 430074, China","institution_ids":["https://openalex.org/I3124059619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137893628","display_name":"Shuyang Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I3124059619","display_name":"China University of Geosciences","ror":"https://ror.org/04gcegc37","country_code":"CN","type":"education","lineage":["https://openalex.org/I3124059619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuyang Liu","raw_affiliation_strings":["School of Computer Science, China University of Geosciences (Wuhan), Wuhan, Hubei, 430074, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, China University of Geosciences (Wuhan), Wuhan, Hubei, 430074, China","institution_ids":["https://openalex.org/I3124059619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137848515","display_name":"Jiahao Zhang (4859203)","orcid":"https://orcid.org/0009-0001-8410-1698"},"institutions":[{"id":"https://openalex.org/I3124059619","display_name":"China University of Geosciences","ror":"https://ror.org/04gcegc37","country_code":"CN","type":"education","lineage":["https://openalex.org/I3124059619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiahao Zhang","raw_affiliation_strings":["School of Computer Science, China University of Geosciences (Wuhan), Wuhan, Hubei, 430074, China"],"raw_orcid":"https://orcid.org/0009-0001-8410-1698","affiliations":[{"raw_affiliation_string":"School of Computer Science, China University of Geosciences (Wuhan), Wuhan, Hubei, 430074, China","institution_ids":["https://openalex.org/I3124059619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137909329","display_name":"Ke Wang","orcid":"https://orcid.org/0009-0006-4316-3231"},"institutions":[{"id":"https://openalex.org/I3124059619","display_name":"China University of Geosciences","ror":"https://ror.org/04gcegc37","country_code":"CN","type":"education","lineage":["https://openalex.org/I3124059619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ke Wang","raw_affiliation_strings":["School of Computer Science, China University of Geosciences (Wuhan), Wuhan, Hubei, 430074, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, China University of Geosciences (Wuhan), Wuhan, Hubei, 430074, China","institution_ids":["https://openalex.org/I3124059619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137828695","display_name":"Chang Tang","orcid":null},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chang Tang","raw_affiliation_strings":["School of Software Engineering, Huazhong University of Science and Technology, Wuhan, Hubei, 430074, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Software Engineering, Huazhong University of Science and Technology, Wuhan, Hubei, 430074, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067600725","display_name":"Dapeng Tao","orcid":"https://orcid.org/0000-0003-0783-5273"},"institutions":[{"id":"https://openalex.org/I189210763","display_name":"Yunnan University","ror":"https://ror.org/0040axw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I189210763"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Dapeng Tao","raw_affiliation_strings":["School of Information Science and Engineering, Yunnan University, Kunming, Yunnan, 650500, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Information Science and Engineering, Yunnan University, Kunming, Yunnan, 650500, China","institution_ids":["https://openalex.org/I189210763"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137832699","display_name":"Zhe Chen","orcid":"https://orcid.org/0000-0001-5004-8975"},"institutions":[{"id":"https://openalex.org/I196829312","display_name":"La Trobe University","ror":"https://ror.org/01rxfrp27","country_code":"AU","type":"education","lineage":["https://openalex.org/I196829312"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Zhe Chen","raw_affiliation_strings":["The School of Computing, Engineering and Mathematical Sciences, La Trobe University, Melbourne, VIC, 3086, Australia"],"raw_orcid":"https://orcid.org/0000-0001-5004-8975","affiliations":[{"raw_affiliation_string":"The School of Computing, Engineering and Mathematical Sciences, La Trobe University, Melbourne, VIC, 3086, Australia","institution_ids":["https://openalex.org/I196829312"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5137873294","display_name":"WEI Xiang","orcid":"https://orcid.org/0000-0002-0608-065X"},"institutions":[{"id":"https://openalex.org/I196829312","display_name":"La Trobe University","ror":"https://ror.org/01rxfrp27","country_code":"AU","type":"education","lineage":["https://openalex.org/I196829312"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Wei Xiang","raw_affiliation_strings":["The School of Computing, Engineering and Mathematical Sciences, La Trobe University, Melbourne, VIC, 3086, Australia"],"raw_orcid":"https://orcid.org/0000-0002-0608-065X","affiliations":[{"raw_affiliation_string":"The School of Computing, Engineering and Mathematical Sciences, La Trobe University, Melbourne, VIC, 3086, Australia","institution_ids":["https://openalex.org/I196829312"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5067600725","https://openalex.org/A5137832699"],"corresponding_institution_ids":["https://openalex.org/I189210763","https://openalex.org/I196829312"],"apc_list":{"value":4360,"currency":"USD","value_usd":4360},"apc_paid":{"value":4360,"currency":"USD","value_usd":4360},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.70365613,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"136","issue":null,"first_page":"104530","last_page":"104530"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9786999821662903,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9786999821662903,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.0027000000700354576,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.0024999999441206455,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.6549000144004822},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.2581000030040741},{"id":"https://openalex.org/keywords/hierarchical-database-model","display_name":"Hierarchical database model","score":0.2531999945640564}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7159000039100647},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.6549000144004822},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3968999981880188},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3806999921798706},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3734999895095825},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2888999879360199},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.2808000147342682},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.2581000030040741},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2540000081062317},{"id":"https://openalex.org/C144986985","wikidata":"https://www.wikidata.org/wiki/Q871236","display_name":"Hierarchical database model","level":2,"score":0.2531999945640564}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1016/j.inffus.2026.104530","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.inffus.2026.104530","pdf_url":null,"source":{"id":"https://openalex.org/S7560371","display_name":"Information Fusion","issn_l":"1566-2535","issn":["1566-2535","1872-6305"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Fusion","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1016/j.inffus.2026.104530","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.inffus.2026.104530","pdf_url":null,"source":{"id":"https://openalex.org/S7560371","display_name":"Information Fusion","issn_l":"1566-2535","issn":["1566-2535","1872-6305"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Fusion","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.4099178612232208,"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322186","display_name":"Natural Science Foundation of Hubei Province","ror":null},{"id":"https://openalex.org/F8678731271","display_name":"Hubei Key Laboratory of Intelligent Geo-Information Processing","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":9,"referenced_works":["https://openalex.org/W2081580037","https://openalex.org/W3003908700","https://openalex.org/W3208639589","https://openalex.org/W4388601298","https://openalex.org/W4399306114","https://openalex.org/W4400849567","https://openalex.org/W4401300783","https://openalex.org/W4402112088","https://openalex.org/W7140396222"],"related_works":[],"abstract_inverted_index":{"Human":[0],"emotion":[1,39,55,92,117,172,204,208],"recognition":[2,56,93],"is":[3],"essential":[4],"for":[5,122,150],"emotionally":[6],"intelligent":[7],"systems":[8],"in":[9,45,82],"areas":[10],"such":[11],"as":[12],"human\u2013computer":[13],"interaction,":[14],"healthcare,":[15],"and":[16,27,85,104,131,169,191,194,206,236,248],"behavioral":[17],"analysis.":[18],"In":[19],"real-world":[20],"scenarios,":[21],"emotional":[22],"states":[23],"are":[24],"often":[25,112],"ambiguous":[26],"fall":[28],"outside":[29],"predefined":[30],"categories,":[31],"making":[32],"existing":[33],"models":[34,61,149],"unable":[35],"to":[36,65,99,114,128,245],"handle":[37],"unseen":[38,106,250],"classes,":[40],"thus":[41],"limiting":[42,86],"their":[43],"use":[44],"sensitive":[46],"domains":[47],"like":[48],"depression":[49],"detection.":[50],"To":[51],"address":[52],"this,":[53],"open-set":[54,90],"(O-ER)":[57],"methods":[58],"with":[59,189,231],"pretrained":[60,148],"have":[62],"been":[63],"proposed":[64],"classify":[66],"known":[67,102,203,247],"emotions":[68,103],"while":[69],"detecting":[70],"unknown":[71,207],"ones.":[72,107],"However,":[73,108],"most":[74],"focus":[75],"on":[76,218],"unimodal":[77],"recognition,":[78],"ignoring":[79],"rich":[80],"cues":[81],"multimodal":[83,91,97,171],"data":[84],"robustness.":[87],"This":[88],"motivates":[89],"(O-MER),":[94],"which":[95],"captures":[96],"information":[98],"better":[100],"recognize":[101,246],"detect":[105,249],"directly":[109],"fusing":[110],"modalities":[111],"leads":[113],"semantically":[115],"heterogeneous":[116],"representations,":[118],"highlighting":[119],"the":[120,229],"need":[121],"a":[123,143,153,211],"unified":[124],"modality-interactive":[125],"prompting":[126],"mechanism":[127],"structurally":[129],"align":[130],"fuse":[132],"cross-modal":[133],"affective":[134],"cues.":[135],"We":[136],"propose":[137],"Hierarchical":[138],"Cross-modal":[139],"Emotion-interactive":[140],"Prompting":[141,166,182],"(HCEP),":[142],"novel":[144,212],"framework":[145],"that":[146,223],"adapts":[147],"O-MER":[151,220],"via":[152,174],"two-level":[154],"prompt":[155],"learning":[156],"mechanism.":[157],"HCEP":[158,224],"includes":[159],"three":[160],"components:":[161],"(1)":[162],"Semantic-level":[163],"Multimodal":[164],"Emotion-aligning":[165],"(SMEP),":[167],"aligning":[168],"capturing":[170],"features":[173],"semantic":[175],"emotion-unified":[176],"prompts;":[177,193],"(2)":[178],"Class-level":[179],"Unimodal":[180],"Emotion-opposing":[181],"(CUEP),":[183],"refining":[184],"decision":[185],"boundaries":[186],"per":[187],"modality":[188],"positive":[190],"negative":[192],"(3)":[195],"Dual-stream":[196],"Prompt-driven":[197],"Open-set":[198],"Learning":[199],"(DPOL),":[200],"jointly":[201],"optimizing":[202],"classification":[205],"detection":[209],"using":[210],"threshold-based":[213],"discrimination":[214],"strategy.":[215],"Extensive":[216],"experiments":[217],"five":[219],"benchmarks":[221],"demonstrate":[222],"achieves":[225],"state-of-the-art":[226],"results,":[227],"outperforming":[228],"baseline":[230],"an":[232],"average":[233],"16.79%":[234],"AUROC":[235],"28.49%":[237],"OSCR":[238],"relative":[239],"gains,":[240],"validating":[241],"its":[242],"strong":[243],"capability":[244],"emotions.":[251]},"counts_by_year":[],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2026-06-05T00:00:00"}