{"id":"https://openalex.org/W4312596733","doi":"https://doi.org/10.1109/icpr56361.2022.9956730","title":"Audio and Video-based Emotion Recognition using Multimodal Transformers","display_name":"Audio and Video-based Emotion Recognition using Multimodal Transformers","publication_year":2022,"publication_date":"2022-08-21","ids":{"openalex":"https://openalex.org/W4312596733","doi":"https://doi.org/10.1109/icpr56361.2022.9956730"},"language":"en","primary_location":{"id":"doi:10.1109/icpr56361.2022.9956730","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icpr56361.2022.9956730","pdf_url":null,"source":{"id":"https://openalex.org/S4363607731","display_name":"2022 26th International Conference on Pattern Recognition (ICPR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 26th International Conference on Pattern Recognition (ICPR)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007962043","display_name":"Vijay John","orcid":"https://orcid.org/0000-0002-9553-0906"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vijay John","raw_affiliation_strings":["Guardian Robot Project, R-IH,RIKEN,Japan","Guardian Robot Project, R-IH, RIKEN, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Guardian Robot Project, R-IH,RIKEN,Japan","institution_ids":[]},{"raw_affiliation_string":"Guardian Robot Project, R-IH, RIKEN, Japan","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027960360","display_name":"Yasutomo Kawanishi","orcid":"https://orcid.org/0000-0002-3799-4550"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yasutomo Kawanishi","raw_affiliation_strings":["Guardian Robot Project, R-IH,RIKEN,Japan","Guardian Robot Project, R-IH, RIKEN, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Guardian Robot Project, R-IH,RIKEN,Japan","institution_ids":[]},{"raw_affiliation_string":"Guardian Robot Project, R-IH, RIKEN, Japan","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.3861,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.97314653,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"2582","last_page":"2588"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8167456388473511},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5818465948104858},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5707058906555176},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5189675688743591},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.48037275671958923},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.42065027356147766},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3427984118461609}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8167456388473511},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5818465948104858},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5707058906555176},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5189675688743591},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.48037275671958923},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.42065027356147766},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3427984118461609},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icpr56361.2022.9956730","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icpr56361.2022.9956730","pdf_url":null,"source":{"id":"https://openalex.org/S4363607731","display_name":"2022 26th International Conference on Pattern Recognition (ICPR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 26th International Conference on Pattern Recognition (ICPR)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5299999713897705,"display_name":"Sustainable cities and communities","id":"https://metadata.un.org/sdg/11"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W1981918162","https://openalex.org/W2030931454","https://openalex.org/W2064675550","https://openalex.org/W2295001676","https://openalex.org/W2399733683","https://openalex.org/W2492473494","https://openalex.org/W2526050071","https://openalex.org/W2548128734","https://openalex.org/W2548442788","https://openalex.org/W2578895956","https://openalex.org/W2610961739","https://openalex.org/W2624340939","https://openalex.org/W2727455004","https://openalex.org/W2745497104","https://openalex.org/W2747664154","https://openalex.org/W2803193013","https://openalex.org/W2892009249","https://openalex.org/W2894871570","https://openalex.org/W2963163009","https://openalex.org/W3007721715","https://openalex.org/W3035558565","https://openalex.org/W3094383023","https://openalex.org/W3094502228","https://openalex.org/W3104377639","https://openalex.org/W3122081138","https://openalex.org/W3137028092","https://openalex.org/W3193714551","https://openalex.org/W4385245566","https://openalex.org/W6727253422","https://openalex.org/W6739901393","https://openalex.org/W6774272549","https://openalex.org/W6779098134","https://openalex.org/W6784333009","https://openalex.org/W6786393251"],"related_works":["https://openalex.org/W2081900870","https://openalex.org/W2037549926","https://openalex.org/W2345479200","https://openalex.org/W2183306018","https://openalex.org/W2849310602","https://openalex.org/W3006008237","https://openalex.org/W2419146053","https://openalex.org/W4388890789","https://openalex.org/W2088247287","https://openalex.org/W2963903416"],"abstract_inverted_index":{"Emotion":[0],"recognition,":[1],"an":[2],"important":[3],"research":[4],"problem":[5],"in":[6,50,89,112,140,169],"human-robot":[7],"interactions,":[8],"is":[9,23,175,196,207],"primarily":[10],"achieved":[11],"by":[12,25],"extracting":[13],"human":[14],"emotions":[15],"from":[16,132,165],"audio":[17,94,114],"and":[18,38,43,99,115,181,190],"visual":[19,162],"data.":[20],"State-of-the-art":[21],"performance":[22,68,139],"reported":[24],"audio-visual":[26,70,183],"sensor":[27],"fusion":[28],"algorithms":[29],"using":[30,73,177],"deep":[31],"learning":[32],"models":[33,195],"such":[34],"as":[35],"CNN,":[36],"RNN,":[37],"LSTM.":[39],"However,":[40],"the":[41,52,56,67,93,96,100,108,113,119,123,137,157,161,166,170,178,202,210],"RNN":[42],"LSTM":[44],"are":[45],"shown":[46],"to":[47,65,155,160],"be":[48],"limited":[49],"handling":[51],"long-term":[53],"dependencies":[54],"over":[55],"entire":[57],"input":[58],"sequence.":[59],"In":[60],"this":[61],"work,":[62,91],"we":[63],"propose":[64,146],"improve":[66],"of":[69],"emotion":[71],"recognition":[72],"a":[74,147],"novel":[75,148],"transformer-based":[76],"model,":[77],"containing":[78],"three":[79,86,134],"transformer":[80,87,205],"branches,":[81,88],"named":[82],"multimodal":[83],"transformers.":[84],"The":[85,104,129,172,198],"our":[90,141],"compute":[92],"self-attention,":[95,98],"video":[97,116],"audio-video":[101,126],"cross":[102],"attention.":[103],"self-attention":[105],"branches":[106,135],"identify":[107],"most":[109,124],"relevant":[110,125,130],"information":[111,131,159],"input,":[117],"while":[118],"cross-attention":[120],"branch":[121],"identifies":[122],"interactive":[127],"information.":[128],"these":[133],"report":[136],"best":[138],"ablation":[142,188],"study.":[143],"We":[144],"also":[145],"temporal":[149,158],"embedding":[150],"scheme,":[151],"termed":[152],"block":[153],"embedding,":[154],"add":[156],"feature,":[163],"derived":[164],"multiple":[167],"frames":[168],"video.":[171],"proposed":[173,203],"architecture":[174],"validated":[176],"RAVDESS,":[179],"CREMA-D,":[180],"SAVEE":[182],"public":[184],"datasets.":[185],"A":[186],"detailed":[187],"study":[189],"comparative":[191],"analysis":[192],"with":[193],"baseline":[194,211],"performed.":[197],"results":[199],"show":[200],"that":[201],"multi-modal":[204],"framework":[206],"better":[208],"than":[209],"methods.":[212]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}