{"id":"https://openalex.org/W3208632377","doi":"https://doi.org/10.1109/icassp43922.2022.9746910","title":"Memobert: Pre-Training Model with Prompt-Based Learning for Multimodal Emotion Recognition","display_name":"Memobert: Pre-Training Model with Prompt-Based Learning for Multimodal Emotion Recognition","publication_year":2022,"publication_date":"2022-04-27","ids":{"openalex":"https://openalex.org/W3208632377","doi":"https://doi.org/10.1109/icassp43922.2022.9746910","mag":"3208632377"},"language":"en","primary_location":{"id":"doi:10.1109/icassp43922.2022.9746910","doi":"https://doi.org/10.1109/icassp43922.2022.9746910","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp43922.2022.9746910","pdf_url":null,"source":{"id":"https://openalex.org/S4363607879","display_name":"IEEE International Conference on Acoustics Speech and Signal Processing","issn_l":"1520-6149","issn":["1520-6149","2379-190X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101428911","display_name":"Jinming Zhao","orcid":"https://orcid.org/0000-0003-0528-6952","observed_orcids":["https://orcid.org/0000-0003-0528-6952"]},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]},{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN","SG"],"is_corresponding":false,"raw_author_name":"Jinming Zhao","raw_affiliation_strings":["Renmin University of China,School of Information","Electrical and Computer Engineering, National University of Singapore","School of Information, Renmin University of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Renmin University of China,School of Information","institution_ids":["https://openalex.org/I78988378"]},{"raw_affiliation_string":"Electrical and Computer Engineering, National University of Singapore","institution_ids":["https://openalex.org/I165932596"]},{"raw_affiliation_string":"School of Information, Renmin University of China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101910099","display_name":"Ruichen Li","orcid":"https://orcid.org/0009-0005-0277-9964","observed_orcids":["https://orcid.org/0009-0005-0277-9964"]},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruichen Li","raw_affiliation_strings":["Renmin University of China,School of Information","School of Information, Renmin University of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Renmin University of China,School of Information","institution_ids":["https://openalex.org/I78988378"]},{"raw_affiliation_string":"School of Information, Renmin University of China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009985839","display_name":"Qin Jin","orcid":"https://orcid.org/0000-0001-6486-6020","observed_orcids":["https://orcid.org/0000-0001-6486-6020"]},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qin Jin","raw_affiliation_strings":["Renmin University of China,School of Information","School of Information, Renmin University of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Renmin University of China,School of Information","institution_ids":["https://openalex.org/I78988378"]},{"raw_affiliation_string":"School of Information, Renmin University of China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015574447","display_name":"Xinchao Wang","orcid":"https://orcid.org/0000-0003-0057-1404","observed_orcids":["https://orcid.org/0000-0003-0057-1404","https://orcid.org/0009-0005-8055-9094"]},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Xinchao Wang","raw_affiliation_strings":["National University of Singapore,Electrical and Computer Engineering","Electrical and Computer Engineering, National University of Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Electrical and Computer Engineering","institution_ids":["https://openalex.org/I165932596"]},{"raw_affiliation_string":"Electrical and Computer Engineering, National University of Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032690182","display_name":"Haizhou Li","orcid":"https://orcid.org/0000-0001-9158-9401","observed_orcids":["https://orcid.org/0000-0001-9158-9401"]},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]},{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN","SG"],"is_corresponding":false,"raw_author_name":"Haizhou Li","raw_affiliation_strings":["National University of Singapore,Electrical and Computer Engineering","Electrical and Computer Engineering, National University of Singapore","The Chinese University of Hong Kong, Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Electrical and Computer Engineering","institution_ids":["https://openalex.org/I165932596"]},{"raw_affiliation_string":"Electrical and Computer Engineering, National University of Singapore","institution_ids":["https://openalex.org/I165932596"]},{"raw_affiliation_string":"The Chinese University of Hong Kong, Shenzhen, China","institution_ids":["https://openalex.org/I4210116924"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":11.7649,"has_fulltext":false,"cited_by_count":39,"citation_normalized_percentile":{"value":0.98757151,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"4703","last_page":"4707"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.003000000026077032,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multimodal-emotion-recognition","display_name":"multimodal emotion recognition","score":0.996999979019165},{"id":"https://openalex.org/keywords/multimodal-pretraining","display_name":"multimodal pre-training","score":0.8460000157356262},{"id":"https://openalex.org/keywords/prompt-learning","display_name":"prompt learning","score":0.871999979019165},{"id":"https://openalex.org/keywords/self-supervised-learning","display_name":"self-supervised learning","score":0.8960000276565552},{"id":"https://openalex.org/keywords/unlabeled-video-data","display_name":"unlabeled video data","score":0.7760000228881836},{"id":"https://openalex.org/keywords/emotion-classification","display_name":"emotion classification","score":0.7620000243186951},{"id":"https://openalex.org/keywords/iemocap","display_name":"IEMOCAP","score":0.871999979019165}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7877578735351562},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.6611708402633667},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6588782668113708},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6548369526863098},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5474052429199219},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.5430211424827576},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.5035547614097595},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.44525179266929626},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4268646836280823},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4075334966182709},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp43922.2022.9746910","doi":"https://doi.org/10.1109/icassp43922.2022.9746910","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp43922.2022.9746910","pdf_url":null,"source":{"id":"https://openalex.org/S4363607879","display_name":"IEEE International Conference on Acoustics Speech and Signal Processing","issn_l":"1520-6149","issn":["1520-6149","2379-190X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":36,"referenced_works":["https://openalex.org/W2032254851","https://openalex.org/W2080830759","https://openalex.org/W2146334809","https://openalex.org/W2245428411","https://openalex.org/W2342475039","https://openalex.org/W2481681431","https://openalex.org/W2787560479","https://openalex.org/W2787581402","https://openalex.org/W2896457183","https://openalex.org/W2947476638","https://openalex.org/W2952370363","https://openalex.org/W2962718314","https://openalex.org/W2963341956","https://openalex.org/W2963446712","https://openalex.org/W2963710346","https://openalex.org/W2969876226","https://openalex.org/W2970231061","https://openalex.org/W2970476646","https://openalex.org/W2981851019","https://openalex.org/W2982223350","https://openalex.org/W3002104146","https://openalex.org/W3035265375","https://openalex.org/W3036601975","https://openalex.org/W3083650259","https://openalex.org/W3090449556","https://openalex.org/W3095379519","https://openalex.org/W3099782249","https://openalex.org/W3105232955","https://openalex.org/W3116459227","https://openalex.org/W3154596443","https://openalex.org/W3160345865","https://openalex.org/W3171654275","https://openalex.org/W3175825020","https://openalex.org/W3185341429","https://openalex.org/W3194309076","https://openalex.org/W6780218876"],"related_works":["https://openalex.org/W4382725068","https://openalex.org/W3193714551","https://openalex.org/W3108792608","https://openalex.org/W4408355306","https://openalex.org/W4405709339","https://openalex.org/W4382491679","https://openalex.org/W4416799710","https://openalex.org/W3083650259","https://openalex.org/W4410454533","https://openalex.org/W7200474254"],"abstract_inverted_index":{"Multimodal":[0],"emotion":[1,39,77,110],"recognition":[2,111],"study":[3],"is":[4],"hindered":[5],"by":[6],"the":[7,20,63,75,87,92],"lack":[8],"of":[9,14],"labelled":[10],"corpora":[11],"in":[12,58],"terms":[13],"scale":[15],"and":[16,24,101],"diversity,":[17],"due":[18],"to":[19,91],"high":[21],"annotation":[22],"cost":[23],"label":[25],"ambiguity.":[26],"In":[27],"this":[28],"paper,":[29],"we":[30,68],"propose":[31,69],"a":[32,50,70,81],"multimodal":[33,38,43],"pre-training":[34],"model":[35],"MEmoBERT":[36,107],"for":[37],"recognition,":[40],"which":[41],"learns":[42],"joint":[44],"representations":[45],"through":[46],"self-supervised":[47],"learning":[48],"from":[49],"self-collected":[51],"large-scale":[52],"unlabeled":[53],"video":[54],"data":[55],"that":[56,73,104],"come":[57],"sheer":[59],"volume.":[60],"Furthermore,":[61],"unlike":[62],"conventional":[64],"\"pre-train,":[65],"finetune\"":[66],"paradigm,":[67],"prompt-based":[71],"method":[72],"reformulates":[74],"downstream":[76,88],"classification":[78],"task":[79,89],"as":[80],"masked":[82],"text":[83],"prediction":[84],"one,":[85],"bringing":[86],"closer":[90],"pre-training.":[93],"Extensive":[94],"experiments":[95],"on":[96],"two":[97],"benchmark":[98],"datasets,":[99],"IEMOCAP":[100],"MSP-IMPROV,":[102],"show":[103],"our":[105],"proposed":[106],"significantly":[108],"enhances":[109],"performance.":[112]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":14},{"year":2022,"cited_by_count":2}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2021-11-08T00:00:00"}