{"id":"https://openalex.org/W4297510826","doi":"https://doi.org/10.1145/3551876.3554813","title":"Leveraging Multi-modal Interactions among the Intermediate Representations of Deep Transformers for Emotion Recognition","display_name":"Leveraging Multi-modal Interactions among the Intermediate Representations of Deep Transformers for Emotion Recognition","publication_year":2022,"publication_date":"2022-09-28","ids":{"openalex":"https://openalex.org/W4297510826","doi":"https://doi.org/10.1145/3551876.3554813"},"language":"en","primary_location":{"id":"doi:10.1145/3551876.3554813","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3551876.3554813","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd International on Multimodal Sentiment Analysis Workshop and Challenge","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061277006","display_name":"Yang Wu","orcid":"https://orcid.org/0000-0003-2197-080X"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yang Wu","raw_affiliation_strings":["Harbin Institute of Technology, Harbin, China"],"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Harbin, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101902939","display_name":"Zhenyu Zhang","orcid":"https://orcid.org/0000-0002-9470-7132"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhenyu Zhang","raw_affiliation_strings":["Harbin Institute of Technology, Harbin, China"],"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Harbin, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103110081","display_name":"Pai Peng","orcid":"https://orcid.org/0000-0002-7129-6174"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pai Peng","raw_affiliation_strings":["Harbin Institute of Technology, Harbin, China"],"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Harbin, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031911989","display_name":"Yanyan Zhao","orcid":"https://orcid.org/0000-0001-5852-7962"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanyan Zhao","raw_affiliation_strings":["Harbin Institute of Technology, Harbin, China"],"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Harbin, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017671620","display_name":"Bing Qin","orcid":"https://orcid.org/0000-0002-2543-5604"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bing Qin","raw_affiliation_strings":["Harbin Institute of Technology, Harbin, China"],"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Harbin, China","institution_ids":["https://openalex.org/I204983213"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5061277006"],"corresponding_institution_ids":["https://openalex.org/I204983213"],"apc_list":null,"apc_paid":null,"fwci":3.3306,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.9240543,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"101","last_page":"109"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.834943413734436},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7262747287750244},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.5827048420906067},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.4931044280529022},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.4754990041255951},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.41913914680480957},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.36347323656082153},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.14226457476615906},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.10649341344833374}],"concepts":[{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.834943413734436},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7262747287750244},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5827048420906067},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.4931044280529022},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4754990041255951},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41913914680480957},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.36347323656082153},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.14226457476615906},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.10649341344833374},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3551876.3554813","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3551876.3554813","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd International on Multimodal Sentiment Analysis Workshop and Challenge","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W1849277567","https://openalex.org/W2085662862","https://openalex.org/W2739862888","https://openalex.org/W2807126412","https://openalex.org/W2883409523","https://openalex.org/W2895368507","https://openalex.org/W2897295818","https://openalex.org/W2948947170","https://openalex.org/W2964051877","https://openalex.org/W2964300796","https://openalex.org/W2982223350","https://openalex.org/W2985882473","https://openalex.org/W3035570025","https://openalex.org/W3093051361","https://openalex.org/W3093434718","https://openalex.org/W3094502228","https://openalex.org/W3099143320","https://openalex.org/W3138516171","https://openalex.org/W3168391820","https://openalex.org/W3169801598","https://openalex.org/W3173549566","https://openalex.org/W3176047188","https://openalex.org/W4226380987","https://openalex.org/W4229948749","https://openalex.org/W4304092044"],"related_works":["https://openalex.org/W4388335561","https://openalex.org/W2970530566","https://openalex.org/W4288261899","https://openalex.org/W4307309205","https://openalex.org/W2967478618","https://openalex.org/W4385009901","https://openalex.org/W4385572700","https://openalex.org/W2997152889","https://openalex.org/W4285141722","https://openalex.org/W4304700937"],"abstract_inverted_index":{"Multi-modal":[0],"emotion":[1,6,63],"recognition":[2],"aims":[3],"to":[4,44,102,120],"recognize":[5],"states":[7],"from":[8,142],"multi-modal":[9,26,38,51,81,86,98,116,124,146],"inputs.":[10],"Existing":[11],"end-to-end":[12,62],"models":[13],"typically":[14],"fuse":[15],"the":[16,20,25,29,37,46,50,54,66,72,80,97,104,108,115,122,138,145,149],"uni-modal":[17],"representations":[18,56],"in":[19],"last":[21],"layers":[22],"without":[23],"leveraging":[24,49],"interactions":[27,52,106,147],"among":[28,53,148],"intermediate":[30,55,150],"representations.":[31,151],"In":[32],"this":[33],"paper,":[34],"we":[35,94,113],"propose":[36],"Recurrent":[39],"Intermediate-Layer":[40],"Aggregation":[41],"(RILA)":[42],"model":[43,70,136],"explore":[45],"effectiveness":[47],"of":[48,57,68,79],"deep":[58],"pre-trained":[59],"transformers":[60],"for":[61],"recognition.":[64],"At":[65],"heart":[67],"our":[69,135],"is":[71],"Intermediate-Representation":[73],"Fusion":[74],"Module":[75],"(IRFM),":[76],"which":[77],"consists":[78],"aggregation":[82,99],"gating":[83,100],"module":[84,101,119],"and":[85,110,131],"token":[87,117],"attention":[88,118],"module.":[89],"Specifically,":[90],"at":[91],"each":[92],"layer,":[93],"first":[95],"use":[96],"capture":[103],"utterance-level":[105],"across":[107],"modalities":[109],"layers.":[111],"Then":[112],"utilize":[114],"leverage":[121],"token-level":[123],"interactions.":[125],"The":[126],"experimental":[127],"results":[128],"on":[129],"IEMOCAP":[130],"CMU-MOSEI":[132],"show":[133],"that":[134],"achieves":[137],"state-of-the-art":[139],"performance,":[140],"benefiting":[141],"fully":[143],"exploiting":[144]},"counts_by_year":[{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}