{"id":"https://openalex.org/W4398226581","doi":"https://doi.org/10.1145/3638682.3638698","title":"A multimodal emotion recognition method based on multiple fusion of audio-visual modalities","display_name":"A multimodal emotion recognition method based on multiple fusion of audio-visual modalities","publication_year":2023,"publication_date":"2023-11-24","ids":{"openalex":"https://openalex.org/W4398226581","doi":"https://doi.org/10.1145/3638682.3638698"},"language":"en","primary_location":{"id":"doi:10.1145/3638682.3638698","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3638682.3638698","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3638682.3638698","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 5th International Conference on Video, Signal and Image Processing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3638682.3638698","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010628229","display_name":"Zheng Wu","orcid":"https://orcid.org/0009-0009-3861-7122"},"institutions":[{"id":"https://openalex.org/I120825670","display_name":"Yunnan Normal University","ror":"https://ror.org/00sc9n023","country_code":"CN","type":"education","lineage":["https://openalex.org/I120825670"]},{"id":"https://openalex.org/I40963666","display_name":"Central China Normal University","ror":"https://ror.org/03x1jna21","country_code":"CN","type":"education","lineage":["https://openalex.org/I40963666"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zheng Wu","raw_affiliation_strings":["Key Laboratory of Education Informatization for Nationalities (Yunnan Normal University), Ministry of Education, China"],"raw_orcid":"https://orcid.org/0009-0009-3861-7122","affiliations":[{"raw_affiliation_string":"Key Laboratory of Education Informatization for Nationalities (Yunnan Normal University), Ministry of Education, China","institution_ids":["https://openalex.org/I120825670","https://openalex.org/I40963666"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084677085","display_name":"Jianhou Gan","orcid":"https://orcid.org/0000-0002-1287-857X"},"institutions":[{"id":"https://openalex.org/I120825670","display_name":"Yunnan Normal University","ror":"https://ror.org/00sc9n023","country_code":"CN","type":"education","lineage":["https://openalex.org/I120825670"]},{"id":"https://openalex.org/I40963666","display_name":"Central China Normal University","ror":"https://ror.org/03x1jna21","country_code":"CN","type":"education","lineage":["https://openalex.org/I40963666"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianhou Gan","raw_affiliation_strings":["Key Laboratory of Education Informatization for Nationalities (Yunnan Normal University), Ministry of Education, China"],"raw_orcid":"https://orcid.org/0000-0002-1287-857X","affiliations":[{"raw_affiliation_string":"Key Laboratory of Education Informatization for Nationalities (Yunnan Normal University), Ministry of Education, China","institution_ids":["https://openalex.org/I120825670","https://openalex.org/I40963666"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008034342","display_name":"Jinsheng Liu","orcid":"https://orcid.org/0009-0003-1731-3863"},"institutions":[{"id":"https://openalex.org/I120825670","display_name":"Yunnan Normal University","ror":"https://ror.org/00sc9n023","country_code":"CN","type":"education","lineage":["https://openalex.org/I120825670"]},{"id":"https://openalex.org/I40963666","display_name":"Central China Normal University","ror":"https://ror.org/03x1jna21","country_code":"CN","type":"education","lineage":["https://openalex.org/I40963666"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jinsheng Liu","raw_affiliation_strings":["Key Laboratory of Education Informatization for Nationalities (Yunnan Normal University), Ministry of Education, China"],"raw_orcid":"https://orcid.org/0009-0003-1731-3863","affiliations":[{"raw_affiliation_string":"Key Laboratory of Education Informatization for Nationalities (Yunnan Normal University), Ministry of Education, China","institution_ids":["https://openalex.org/I120825670","https://openalex.org/I40963666"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027553944","display_name":"Jun Wang","orcid":"https://orcid.org/0000-0001-9878-4664"},"institutions":[{"id":"https://openalex.org/I120825670","display_name":"Yunnan Normal University","ror":"https://ror.org/00sc9n023","country_code":"CN","type":"education","lineage":["https://openalex.org/I120825670"]},{"id":"https://openalex.org/I40963666","display_name":"Central China Normal University","ror":"https://ror.org/03x1jna21","country_code":"CN","type":"education","lineage":["https://openalex.org/I40963666"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Wang","raw_affiliation_strings":["Key Laboratory of Education Informatization for Nationalities (Yunnan Normal University), Ministry of Education, China"],"raw_orcid":"https://orcid.org/0000-0001-9878-4664","affiliations":[{"raw_affiliation_string":"Key Laboratory of Education Informatization for Nationalities (Yunnan Normal University), Ministry of Education, China","institution_ids":["https://openalex.org/I120825670","https://openalex.org/I40963666"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.30506391,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"108","last_page":"114"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.991599977016449,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.9456329345703125},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.745337724685669},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.6249540448188782},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.611038863658905},{"id":"https://openalex.org/keywords/facial-expression","display_name":"Facial expression","score":0.5524017810821533},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5306181907653809},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.520102858543396},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5138804912567139},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.4980461597442627},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"Multimodal learning","score":0.47551828622817993},{"id":"https://openalex.org/keywords/fusion","display_name":"Fusion","score":0.4393211901187897},{"id":"https://openalex.org/keywords/sensor-fusion","display_name":"Sensor fusion","score":0.42282021045684814},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.40311288833618164}],"concepts":[{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.9456329345703125},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.745337724685669},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.6249540448188782},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.611038863658905},{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.5524017810821533},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5306181907653809},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.520102858543396},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5138804912567139},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.4980461597442627},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.47551828622817993},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.4393211901187897},{"id":"https://openalex.org/C33954974","wikidata":"https://www.wikidata.org/wiki/Q486494","display_name":"Sensor fusion","level":2,"score":0.42282021045684814},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.40311288833618164},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3638682.3638698","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3638682.3638698","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3638682.3638698","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 5th International Conference on Video, Signal and Image Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3638682.3638698","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3638682.3638698","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3638682.3638698","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 5th International Conference on Video, Signal and Image Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.5600000023841858,"display_name":"No poverty","id":"https://metadata.un.org/sdg/1"}],"awards":[{"id":"https://openalex.org/G1361017843","display_name":null,"funder_award_id":"No. 62166050","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5357890769","display_name":null,"funder_award_id":"62166050","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320327681","display_name":"Yunnan Normal University","ror":"https://ror.org/00sc9n023"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4398226581.pdf","grobid_xml":"https://content.openalex.org/works/W4398226581.grobid-xml"},"referenced_works_count":32,"referenced_works":["https://openalex.org/W283543405","https://openalex.org/W1844030040","https://openalex.org/W2123442489","https://openalex.org/W2154024118","https://openalex.org/W2295001676","https://openalex.org/W2605133118","https://openalex.org/W2618530766","https://openalex.org/W2625297138","https://openalex.org/W2707551695","https://openalex.org/W2750779823","https://openalex.org/W2766028522","https://openalex.org/W2803193013","https://openalex.org/W2805005636","https://openalex.org/W2883409523","https://openalex.org/W2897191477","https://openalex.org/W2951124019","https://openalex.org/W2964010806","https://openalex.org/W2972463723","https://openalex.org/W2981171802","https://openalex.org/W2990276885","https://openalex.org/W2996799978","https://openalex.org/W3014556410","https://openalex.org/W3034849760","https://openalex.org/W3090471441","https://openalex.org/W3175546442","https://openalex.org/W3186949302","https://openalex.org/W3213879871","https://openalex.org/W4205633160","https://openalex.org/W4221161688","https://openalex.org/W4281701185","https://openalex.org/W6763650382","https://openalex.org/W7009639159"],"related_works":["https://openalex.org/W2584926856","https://openalex.org/W2075935902","https://openalex.org/W2014713986","https://openalex.org/W4381827277","https://openalex.org/W3157841754","https://openalex.org/W4390136517","https://openalex.org/W3167558523","https://openalex.org/W3120825179","https://openalex.org/W2014028898","https://openalex.org/W3047624172"],"abstract_inverted_index":{"Human":[0],"emotions":[1],"are":[2,22,58,156],"usually":[3],"expressed":[4],"in":[5,16,28,36,84,125,151],"multiple":[6],"ways,":[7],"including":[8],"speech,":[9],"facial":[10,73],"expressions,":[11],"body":[12],"language,":[13],"etc.":[14],"However,":[15],"multimodal":[17],"fusion":[18,53,94,130],"emotion":[19,44,64],"recognition,":[20],"there":[21],"often":[23],"challenges":[24],"such":[25],"as":[26,77],"difficulty":[27],"data":[29],"alignment,":[30],"significant":[31,48],"feature":[32,49],"heterogeneity,":[33],"and":[34,51,75,79,122,145],"differences":[35],"the":[37,47,96,104,107,111,126],"overall":[38],"contribution":[39],"of":[40,71,98,113],"different":[41,56,92],"modalities":[42,57,70,115,155],"to":[43,101,116,119,142],"recognition.":[45],"Considering":[46],"heterogeneity":[50],"poor":[52],"effect":[54],"when":[55],"merged,":[59],"this":[60],"paper":[61],"proposes":[62],"an":[63,85],"recognition":[65],"model":[66,89],"that":[67,134],"selects":[68],"two":[69,91,108,136],"human":[72],"expressions":[74],"speech":[76],"input":[78],"can":[80,146],"solve":[81],"these":[82,135],"problems":[83],"end-to-end":[86],"manner.":[87],"The":[88],"contains":[90],"modal":[93],"mechanisms,":[95],"core":[97],"which":[99],"is":[100],"focus":[102],"on":[103],"interaction":[105],"between":[106],"modalities,":[109],"allowing":[110],"features":[112],"both":[114],"better":[117,129],"adapt":[118],"each":[120],"other,":[121],"be":[123],"re-fused":[124],"architecture":[127],"for":[128],"effect.":[131],"We":[132],"found":[133],"models":[137],"significantly":[138],"improve":[139],"performance":[140,150],"compared":[141],"previous":[143],"methods":[144],"still":[147],"maintain":[148],"good":[149],"scenarios":[152],"where":[153],"certain":[154],"missing.":[157]},"counts_by_year":[],"updated_date":"2026-07-31T08:31:51.225901","created_date":"2025-10-10T00:00:00"}