{"id":"https://openalex.org/W3088631780","doi":"https://doi.org/10.1109/access.2020.3026823","title":"Multimodal Emotion Recognition With Transformer-Based Self Supervised Feature Fusion","display_name":"Multimodal Emotion Recognition With Transformer-Based Self Supervised Feature Fusion","publication_year":2020,"publication_date":"2020-01-01","ids":{"openalex":"https://openalex.org/W3088631780","doi":"https://doi.org/10.1109/access.2020.3026823","mag":"3088631780"},"language":"en","primary_location":{"id":"doi:10.1109/access.2020.3026823","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2020.3026823","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09206016.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09206016.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072026088","display_name":"Shamane Siriwardhana","orcid":"https://orcid.org/0000-0003-0688-7901"},"institutions":[{"id":"https://openalex.org/I154130895","display_name":"University of Auckland","ror":"https://ror.org/03b94tp07","country_code":"NZ","type":"education","lineage":["https://openalex.org/I154130895"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"Shamane Siriwardhana","raw_affiliation_strings":["Augmented Human Laboratory, Auckland Bioengineering Institute, The University of Auckland, Auckland, New Zealand"],"raw_orcid":"https://orcid.org/0000-0003-0688-7901","affiliations":[{"raw_affiliation_string":"Augmented Human Laboratory, Auckland Bioengineering Institute, The University of Auckland, Auckland, New Zealand","institution_ids":["https://openalex.org/I154130895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001930124","display_name":"Tharindu Kaluarachchi","orcid":"https://orcid.org/0000-0002-1777-231X"},"institutions":[{"id":"https://openalex.org/I154130895","display_name":"University of Auckland","ror":"https://ror.org/03b94tp07","country_code":"NZ","type":"education","lineage":["https://openalex.org/I154130895"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"Tharindu Kaluarachchi","raw_affiliation_strings":["Augmented Human Laboratory, Auckland Bioengineering Institute, The University of Auckland, Auckland, New Zealand"],"raw_orcid":"https://orcid.org/0000-0002-1777-231X","affiliations":[{"raw_affiliation_string":"Augmented Human Laboratory, Auckland Bioengineering Institute, The University of Auckland, Auckland, New Zealand","institution_ids":["https://openalex.org/I154130895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021195349","display_name":"Mark Billinghurst","orcid":"https://orcid.org/0000-0003-4172-6759"},"institutions":[{"id":"https://openalex.org/I154130895","display_name":"University of Auckland","ror":"https://ror.org/03b94tp07","country_code":"NZ","type":"education","lineage":["https://openalex.org/I154130895"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"Mark Billinghurst","raw_affiliation_strings":["Empathic Computing Laboratory, Auckland Bioengineering Institute, The University of Auckland, Auckland, New Zealand"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Empathic Computing Laboratory, Auckland Bioengineering Institute, The University of Auckland, Auckland, New Zealand","institution_ids":["https://openalex.org/I154130895"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027647989","display_name":"Suranga Nanayakkara","orcid":"https://orcid.org/0000-0001-7441-5493"},"institutions":[{"id":"https://openalex.org/I154130895","display_name":"University of Auckland","ror":"https://ror.org/03b94tp07","country_code":"NZ","type":"education","lineage":["https://openalex.org/I154130895"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"Suranga Nanayakkara","raw_affiliation_strings":["Augmented Human Laboratory, Auckland Bioengineering Institute, The University of Auckland, Auckland, New Zealand"],"raw_orcid":"https://orcid.org/0000-0001-7441-5493","affiliations":[{"raw_affiliation_string":"Augmented Human Laboratory, Auckland Bioengineering Institute, The University of Auckland, Auckland, New Zealand","institution_ids":["https://openalex.org/I154130895"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I154130895"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":12.5647,"has_fulltext":true,"cited_by_count":154,"citation_normalized_percentile":{"value":0.99091662,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"8","issue":null,"first_page":"176274","last_page":"176285"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8075960874557495},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.7080779671669006},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"Multimodal learning","score":0.5921562910079956},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5917145013809204},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.5848944187164307},{"id":"https://openalex.org/keywords/feature-learning","display_name":"Feature learning","score":0.5627181529998779},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4517672061920166},{"id":"https://openalex.org/keywords/facial-expression","display_name":"Facial expression","score":0.41945379972457886},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.4139247536659241},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.41376790404319763},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4069637060165405},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3952144384384155}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8075960874557495},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.7080779671669006},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.5921562910079956},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5917145013809204},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.5848944187164307},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.5627181529998779},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4517672061920166},{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.41945379972457886},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4139247536659241},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.41376790404319763},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4069637060165405},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3952144384384155},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/access.2020.3026823","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2020.3026823","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09206016.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:ac896cc9ea6b4326b2621a61c8fd9f63","is_oa":true,"landing_page_url":"https://doaj.org/article/ac896cc9ea6b4326b2621a61c8fd9f63","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 8, Pp 176274-176285 (2020)","raw_type":"article"},{"id":"pmh:oai:researchspace.auckland.ac.nz:2292/53886","is_oa":true,"landing_page_url":"https://hdl.handle.net/2292/53886","pdf_url":null,"source":{"id":"https://openalex.org/S7407055463","display_name":"ResearchSpace (University of Auckland)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I154130895","host_organization_name":"University of Auckland","host_organization_lineage":["https://openalex.org/I154130895"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Journal Article"}],"best_oa_location":{"id":"doi:10.1109/access.2020.3026823","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2020.3026823","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09206016.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.7200000286102295,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3088631780.pdf","grobid_xml":"https://content.openalex.org/works/W3088631780.grobid-xml"},"referenced_works_count":75,"referenced_works":["https://openalex.org/W343636949","https://openalex.org/W1494198834","https://openalex.org/W1566289585","https://openalex.org/W1595126664","https://openalex.org/W2074788634","https://openalex.org/W2095176743","https://openalex.org/W2146334809","https://openalex.org/W2250539671","https://openalex.org/W2253728219","https://openalex.org/W2339343773","https://openalex.org/W2465534249","https://openalex.org/W2769112066","https://openalex.org/W2769114491","https://openalex.org/W2769857323","https://openalex.org/W2775666429","https://openalex.org/W2784665486","https://openalex.org/W2808631503","https://openalex.org/W2842511635","https://openalex.org/W2883409523","https://openalex.org/W2887997593","https://openalex.org/W2896457183","https://openalex.org/W2898651222","https://openalex.org/W2913939497","https://openalex.org/W2914132458","https://openalex.org/W2923014074","https://openalex.org/W2933138175","https://openalex.org/W2943235166","https://openalex.org/W2954345346","https://openalex.org/W2963103975","https://openalex.org/W2963182768","https://openalex.org/W2963310665","https://openalex.org/W2963341956","https://openalex.org/W2963403868","https://openalex.org/W2963533390","https://openalex.org/W2963686995","https://openalex.org/W2963710346","https://openalex.org/W2964051877","https://openalex.org/W2965373594","https://openalex.org/W2966715458","https://openalex.org/W2969958763","https://openalex.org/W2970608575","https://openalex.org/W2971250720","https://openalex.org/W2971671202","https://openalex.org/W2973049979","https://openalex.org/W2980927909","https://openalex.org/W2985882473","https://openalex.org/W3007708573","https://openalex.org/W3014475539","https://openalex.org/W3015489952","https://openalex.org/W3023371261","https://openalex.org/W3030163527","https://openalex.org/W3037572520","https://openalex.org/W3205498744","https://openalex.org/W4230277160","https://openalex.org/W4292779060","https://openalex.org/W4297808394","https://openalex.org/W4385245566","https://openalex.org/W6719667659","https://openalex.org/W6739901393","https://openalex.org/W6745709488","https://openalex.org/W6748726628","https://openalex.org/W6753277404","https://openalex.org/W6754988648","https://openalex.org/W6755207826","https://openalex.org/W6755541679","https://openalex.org/W6756244841","https://openalex.org/W6758354414","https://openalex.org/W6761937618","https://openalex.org/W6766673545","https://openalex.org/W6766904570","https://openalex.org/W6767101625","https://openalex.org/W6767305871","https://openalex.org/W6769352634","https://openalex.org/W6769481939","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W2962931510","https://openalex.org/W4380551887","https://openalex.org/W2904518532","https://openalex.org/W4285159263","https://openalex.org/W4280529741","https://openalex.org/W4293919860","https://openalex.org/W2963650472","https://openalex.org/W2556013083","https://openalex.org/W3199271201","https://openalex.org/W2584926856"],"abstract_inverted_index":{"Emotion":[0],"Recognition":[1],"is":[2,143],"a":[3,47,108],"challenging":[4],"research":[5,51],"area":[6],"given":[7],"its":[8],"complex":[9],"nature,":[10],"and":[11,24,27,49,85,111,121,134,145],"humans":[12],"express":[13],"emotional":[14],"cues":[15],"across":[16],"various":[17],"modalities":[18,80],"such":[19],"as":[20],"language,":[21],"facial":[22],"expressions,":[23],"speech.":[25],"Representation":[26],"fusion":[28,113],"of":[29,81,103,128],"features":[30,88,120],"are":[31],"the":[32,70,74,99,126,147],"most":[33],"crucial":[34],"tasks":[35],"in":[36,53,73,95],"multimodal":[37,118,129],"emotion":[38,130],"recognition":[39],"research.":[40],"Self":[41],"Supervised":[42],"Learning":[43],"(SSL)":[44],"has":[45],"become":[46],"prominent":[48],"influential":[50],"direction":[52],"representation":[54],"learning,":[55],"where":[56],"researchers":[57],"have":[58],"access":[59],"to":[60,138],"pre-trained":[61,92],"SSL":[62,93,104,119],"models":[63,94,149],"that":[64,115,140],"represent":[65,77],"different":[66],"data":[67],"modalities.":[68],"For":[69],"first":[71],"time":[72],"literature,":[75],"we":[76,106],"three":[78],"input":[79],"text,":[82],"audio":[83],"(speech),":[84],"vision":[86],"with":[87],"extracted":[89],"from":[90],"independently":[91],"this":[96],"paper.":[97],"Given":[98],"high":[100],"dimensional":[101],"nature":[102],"features,":[105],"introduce":[107],"novel":[109],"Transformers":[110],"Attention-based":[112],"mechanism":[114],"can":[116],"combine":[117],"achieve":[122],"state-of-the-art":[123,148],"results":[124],"for":[125],"task":[127],"recognition.":[131],"We":[132],"benchmark":[133],"evaluate":[135],"our":[136,141],"work":[137],"show":[139],"model":[142],"robust":[144],"outperforms":[146],"on":[150],"four":[151],"datasets.":[152]},"counts_by_year":[{"year":2026,"cited_by_count":12},{"year":2025,"cited_by_count":33},{"year":2024,"cited_by_count":36},{"year":2023,"cited_by_count":40},{"year":2022,"cited_by_count":17},{"year":2021,"cited_by_count":16}],"updated_date":"2026-07-14T08:27:34.040176","created_date":"2025-10-10T00:00:00"}