{"id":"https://openalex.org/W4224916926","doi":"https://doi.org/10.1109/icassp43922.2022.9747723","title":"Multimodal Transformer with Learnable Frontend and Self Attention for Emotion Recognition","display_name":"Multimodal Transformer with Learnable Frontend and Self Attention for Emotion Recognition","publication_year":2022,"publication_date":"2022-04-27","ids":{"openalex":"https://openalex.org/W4224916926","doi":"https://doi.org/10.1109/icassp43922.2022.9747723"},"language":"en","primary_location":{"id":"doi:10.1109/icassp43922.2022.9747723","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp43922.2022.9747723","pdf_url":null,"source":{"id":"https://openalex.org/S4363607702","display_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5035530630","display_name":"Soumya Dutta","orcid":"https://orcid.org/0000-0002-3936-6718"},"institutions":[{"id":"https://openalex.org/I59270414","display_name":"Indian Institute of Science Bangalore","ror":"https://ror.org/04dese585","country_code":"IN","type":"education","lineage":["https://openalex.org/I59270414"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Soumya Dutta","raw_affiliation_strings":["Indian Institute of Science,Learning and Extraction of Acoustic Patterns (LEAP) Lab,Bangalore","Learning and Extraction of Acoustic Patterns (LEAP) Lab, Indian Institute of Science, Bangalore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Science,Learning and Extraction of Acoustic Patterns (LEAP) Lab,Bangalore","institution_ids":["https://openalex.org/I59270414"]},{"raw_affiliation_string":"Learning and Extraction of Acoustic Patterns (LEAP) Lab, Indian Institute of Science, Bangalore","institution_ids":["https://openalex.org/I59270414"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5002536077","display_name":"Sriram Ganapathy","orcid":"https://orcid.org/0000-0002-5779-9066"},"institutions":[{"id":"https://openalex.org/I59270414","display_name":"Indian Institute of Science Bangalore","ror":"https://ror.org/04dese585","country_code":"IN","type":"education","lineage":["https://openalex.org/I59270414"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Sriram Ganapathy","raw_affiliation_strings":["Indian Institute of Science,Learning and Extraction of Acoustic Patterns (LEAP) Lab,Bangalore","Learning and Extraction of Acoustic Patterns (LEAP) Lab, Indian Institute of Science, Bangalore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Science,Learning and Extraction of Acoustic Patterns (LEAP) Lab,Bangalore","institution_ids":["https://openalex.org/I59270414"]},{"raw_affiliation_string":"Learning and Extraction of Acoustic Patterns (LEAP) Lab, Indian Institute of Science, Bangalore","institution_ids":["https://openalex.org/I59270414"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I59270414"],"apc_list":null,"apc_paid":null,"fwci":7.5384,"has_fulltext":false,"cited_by_count":28,"citation_normalized_percentile":{"value":0.97866745,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"6917","last_page":"6921"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7769179344177246},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6912392973899841},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6904104948043823},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.6344194412231445},{"id":"https://openalex.org/keywords/utterance","display_name":"Utterance","score":0.6094530820846558},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.5895612835884094},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.49079999327659607},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.44523555040359497},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.43289294838905334},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.42323893308639526},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.41168636083602905},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10442537069320679},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.08561760187149048}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7769179344177246},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6912392973899841},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6904104948043823},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.6344194412231445},{"id":"https://openalex.org/C2775852435","wikidata":"https://www.wikidata.org/wiki/Q258403","display_name":"Utterance","level":2,"score":0.6094530820846558},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.5895612835884094},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.49079999327659607},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44523555040359497},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.43289294838905334},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.42323893308639526},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.41168636083602905},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10442537069320679},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.08561760187149048},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/icassp43922.2022.9747723","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp43922.2022.9747723","pdf_url":null,"source":{"id":"https://openalex.org/S4363607702","display_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},{"id":"pmh:oai::75355","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4377196309","display_name":"NOT FOUND REPOSITORY (Indian Institute of Science Bangalore)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I59270414","host_organization_name":"Indian Institute of Science Bangalore","host_organization_lineage":["https://openalex.org/I59270414"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Conference Paper"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.6499999761581421}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":40,"referenced_works":["https://openalex.org/W1002150248","https://openalex.org/W1542637018","https://openalex.org/W1555767263","https://openalex.org/W1578859036","https://openalex.org/W2009059481","https://openalex.org/W2028975938","https://openalex.org/W2079725295","https://openalex.org/W2085662862","https://openalex.org/W2143492886","https://openalex.org/W2146334809","https://openalex.org/W2156196970","https://openalex.org/W2157331557","https://openalex.org/W2250539671","https://openalex.org/W2544981553","https://openalex.org/W2584561145","https://openalex.org/W2602034649","https://openalex.org/W2740550900","https://openalex.org/W2774090594","https://openalex.org/W2896457183","https://openalex.org/W2937584914","https://openalex.org/W2962770129","https://openalex.org/W2963104701","https://openalex.org/W2963702064","https://openalex.org/W2963800675","https://openalex.org/W2964052309","https://openalex.org/W2966518489","https://openalex.org/W2997258743","https://openalex.org/W3015988193","https://openalex.org/W3095379519","https://openalex.org/W3096723250","https://openalex.org/W3190171050","https://openalex.org/W4287372095","https://openalex.org/W4294367149","https://openalex.org/W4299280181","https://openalex.org/W4385245566","https://openalex.org/W6739901393","https://openalex.org/W6747248625","https://openalex.org/W6750449527","https://openalex.org/W6755207826","https://openalex.org/W6789403026"],"related_works":["https://openalex.org/W2529301793","https://openalex.org/W2384121599","https://openalex.org/W2562096895","https://openalex.org/W2333799855","https://openalex.org/W3177678247","https://openalex.org/W1999617572","https://openalex.org/W3016124757","https://openalex.org/W3034520363","https://openalex.org/W3105646692","https://openalex.org/W3212086526"],"abstract_inverted_index":{"In":[0],"this":[1],"work,":[2],"we":[3,105,139],"propose":[4],"a":[5,25,33,52,67,81],"novel":[6],"approach":[7],"for":[8],"multi-modal":[9,75],"emotion":[10,130],"recognition":[11,131,156],"from":[12,42,47],"conversations":[13],"using":[14,66,80],"speech":[15,155],"and":[16,60,89],"text.":[17],"The":[18,37,96],"audio":[19,27,61,90,136],"representations":[20,39,46,62],"are":[21,40,63,98],"learned":[22],"jointly":[23],"with":[24,51,71,86],"learnable":[26],"front-end":[28],"(LEAF)":[29],"model":[30,144],"feeding":[31],"to":[32,127,148],"CNN":[34],"based":[35],"classifier.":[36],"text":[38],"derived":[41],"pre-trained":[43],"bidirectional":[44,68],"encoder":[45],"transformer":[48,82],"(BERT)":[49],"along":[50],"gated":[53],"recurrent":[54],"network":[55,70],"(GRU).":[56],"Both":[57],"the":[58,74,87,93,101,108,113,119,128,135,143],"textual":[59,88,149],"separately":[64],"processed":[65],"GRU":[69],"self-attention.":[72],"Further,":[73,138],"information":[76],"extraction":[77],"is":[78,84,124,145],"achieved":[79,133],"that":[83,107,142],"input":[85],"embeddings":[91],"at":[92],"utterance":[94],"level.":[95],"experiments":[97],"performed":[99],"on":[100],"IEMOCAP":[102],"database,":[103],"where":[104],"show":[106,141],"proposed":[109],"framework":[110],"improves":[111],"over":[112],"current":[114],"state-of-the-art":[115],"results":[116],"under":[117],"all":[118],"common":[120],"test":[121],"settings.":[122],"This":[123],"primarily":[125],"due":[126],"improved":[129],"performance":[132],"in":[134],"domain.":[137],"also":[140],"more":[146],"robust":[147],"errors":[150],"caused":[151],"by":[152],"an":[153],"automatic":[154],"(ASR)":[157],"system.":[158]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}