{"id":"https://openalex.org/W4362496488","doi":"https://doi.org/10.1109/taffc.2023.3263907","title":"Transformer-Based Self-Supervised Multimodal Representation Learning for Wearable Emotion Recognition","display_name":"Transformer-Based Self-Supervised Multimodal Representation Learning for Wearable Emotion Recognition","publication_year":2023,"publication_date":"2023-04-03","ids":{"openalex":"https://openalex.org/W4362496488","doi":"https://doi.org/10.1109/taffc.2023.3263907"},"language":"en","primary_location":{"id":"doi:10.1109/taffc.2023.3263907","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2023.3263907","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://hal.science/hal-04051433v1/document","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5066217049","display_name":"Yujin Wu","orcid":"https://orcid.org/0000-0002-9932-9409"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I2279609970","display_name":"Universit\u00e9 de Lille","ror":"https://ror.org/02kzqn938","country_code":"FR","type":"education","lineage":["https://openalex.org/I2279609970"]},{"id":"https://openalex.org/I7454413","display_name":"\u00c9cole Centrale de Lille","ror":"https://ror.org/01x441g73","country_code":"FR","type":"education","lineage":["https://openalex.org/I7454413"]}],"countries":["FR"],"is_corresponding":true,"raw_author_name":"Yujin Wu","raw_affiliation_strings":["Univ. Lille, CNRS, Centrale Lille, UMR 9189 CRIStAL, Lille, France"],"raw_orcid":"https://orcid.org/0000-0002-9932-9409","affiliations":[{"raw_affiliation_string":"Univ. Lille, CNRS, Centrale Lille, UMR 9189 CRIStAL, Lille, France","institution_ids":["https://openalex.org/I7454413","https://openalex.org/I2279609970","https://openalex.org/I1294671590"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017977139","display_name":"Mohamed Daoudi","orcid":"https://orcid.org/0000-0003-4219-7860"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I2279609970","display_name":"Universit\u00e9 de Lille","ror":"https://ror.org/02kzqn938","country_code":"FR","type":"education","lineage":["https://openalex.org/I2279609970"]},{"id":"https://openalex.org/I7454413","display_name":"\u00c9cole Centrale de Lille","ror":"https://ror.org/01x441g73","country_code":"FR","type":"education","lineage":["https://openalex.org/I7454413"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Mohamed Daoudi","raw_affiliation_strings":["Univ. Lille, CNRS, Centrale Lille, Institut Mines-Télécom, UMR 9189 CRIStAL, Lille, France"],"raw_orcid":"https://orcid.org/0000-0003-4219-7860","affiliations":[{"raw_affiliation_string":"Univ. Lille, CNRS, Centrale Lille, Institut Mines-Télécom, UMR 9189 CRIStAL, Lille, France","institution_ids":["https://openalex.org/I2279609970","https://openalex.org/I7454413","https://openalex.org/I1294671590"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036770387","display_name":"Ali Amad","orcid":"https://orcid.org/0000-0002-9029-2910"},"institutions":[{"id":"https://openalex.org/I154526488","display_name":"Inserm","ror":"https://ror.org/02vjkv261","country_code":"FR","type":"government","lineage":["https://openalex.org/I154526488"]},{"id":"https://openalex.org/I2279609970","display_name":"Universit\u00e9 de Lille","ror":"https://ror.org/02kzqn938","country_code":"FR","type":"education","lineage":["https://openalex.org/I2279609970"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Ali Amad","raw_affiliation_strings":["Lille Neuroscience & Cognition U1172 INSERM labotory, University of Lille, Lille, France"],"raw_orcid":"https://orcid.org/0000-0002-9029-2910","affiliations":[{"raw_affiliation_string":"Lille Neuroscience & Cognition U1172 INSERM labotory, University of Lille, Lille, France","institution_ids":["https://openalex.org/I154526488","https://openalex.org/I2279609970"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5066217049"],"corresponding_institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I2279609970","https://openalex.org/I7454413"],"apc_list":null,"apc_paid":null,"fwci":22.0632,"has_fulltext":true,"cited_by_count":91,"citation_normalized_percentile":{"value":0.99663309,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"15","issue":"1","first_page":"157","last_page":"172"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10429","display_name":"EEG and Brain-Computer Interfaces","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.968500018119812,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.756038248538971},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.615066409111023},{"id":"https://openalex.org/keywords/overfitting","display_name":"Overfitting","score":0.5810175538063049},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.4632737636566162},{"id":"https://openalex.org/keywords/wearable-computer","display_name":"Wearable computer","score":0.4615843892097473},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.45975399017333984},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.45512861013412476},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.42686140537261963},{"id":"https://openalex.org/keywords/autoencoder","display_name":"Autoencoder","score":0.418460875749588},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.40946683287620544},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.2534991502761841},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.20338767766952515}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.756038248538971},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.615066409111023},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.5810175538063049},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.4632737636566162},{"id":"https://openalex.org/C150594956","wikidata":"https://www.wikidata.org/wiki/Q1334829","display_name":"Wearable computer","level":2,"score":0.4615843892097473},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45975399017333984},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.45512861013412476},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.42686140537261963},{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.418460875749588},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.40946683287620544},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2534991502761841},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.20338767766952515},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/taffc.2023.3263907","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2023.3263907","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},{"id":"pmh:oai:HAL:hal-04051433v1","is_oa":true,"landing_page_url":"https://hal.science/hal-04051433","pdf_url":"https://hal.science/hal-04051433v1/document","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Transactions on Affective Computing, 2024, 15 (1), pp.157-172. ⟨10.1109/TAFFC.2023.3263907⟩","raw_type":"Journal articles"},{"id":"pmh:oai:lilloa.univ-lille.fr:20.500.12210/114218","is_oa":true,"landing_page_url":"http://hdl.handle.net/20.500.12210/114218","pdf_url":null,"source":{"id":"https://openalex.org/S4306402203","display_name":"LillOA (Universit\u00e9 de Lille (University Of Lille))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210123514","host_organization_name":"Centre d'Etudes en Civilisations, Langues et Litt\u00e9ratures Etrang\u00e8res","host_organization_lineage":["https://openalex.org/I4210123514"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:lilloa.univ-lille.fr:20.500.12210/79792","is_oa":true,"landing_page_url":"http://hdl.handle.net/20.500.12210/79792","pdf_url":null,"source":{"id":"https://openalex.org/S4306402203","display_name":"LillOA (Universit\u00e9 de Lille (University Of Lille))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210123514","host_organization_name":"Centre d'Etudes en Civilisations, Langues et Litt\u00e9ratures Etrang\u00e8res","host_organization_lineage":["https://openalex.org/I4210123514"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:oai:HAL:hal-04051433v1","is_oa":true,"landing_page_url":"https://hal.science/hal-04051433","pdf_url":"https://hal.science/hal-04051433v1/document","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Transactions on Affective Computing, 2024, 15 (1), pp.157-172. ⟨10.1109/TAFFC.2023.3263907⟩","raw_type":"Journal articles"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4859330304","display_name":null,"funder_award_id":"ANR-16-IDEX-0004 ULNE","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G6942815888","display_name":null,"funder_award_id":"16-IDEX-0004","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G7225929873","display_name":null,"funder_award_id":"ANR-16-IDEX-0004","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"}],"funders":[{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"},{"id":"https://openalex.org/F4320338463","display_name":"CHIST-ERA","ror":"https://ror.org/00rbzpz17"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4362496488.pdf","grobid_xml":"https://content.openalex.org/works/W4362496488.grobid-xml"},"referenced_works_count":54,"referenced_works":["https://openalex.org/W2046556084","https://openalex.org/W2159872201","https://openalex.org/W2167801237","https://openalex.org/W2171801645","https://openalex.org/W2184481998","https://openalex.org/W2194775991","https://openalex.org/W2598291992","https://openalex.org/W2620664872","https://openalex.org/W2792764867","https://openalex.org/W2894771803","https://openalex.org/W2896457183","https://openalex.org/W2901027086","https://openalex.org/W2908671501","https://openalex.org/W2938004196","https://openalex.org/W2964051877","https://openalex.org/W2973049979","https://openalex.org/W2979903110","https://openalex.org/W2981372722","https://openalex.org/W2997026866","https://openalex.org/W2999294457","https://openalex.org/W3005055041","https://openalex.org/W3005680577","https://openalex.org/W3021080764","https://openalex.org/W3025630664","https://openalex.org/W3035164673","https://openalex.org/W3035725276","https://openalex.org/W3040854424","https://openalex.org/W3046296398","https://openalex.org/W3099025572","https://openalex.org/W3101658985","https://openalex.org/W3101667008","https://openalex.org/W3103291722","https://openalex.org/W3116273309","https://openalex.org/W3125128672","https://openalex.org/W3128352929","https://openalex.org/W3177342940","https://openalex.org/W3188872815","https://openalex.org/W3196418137","https://openalex.org/W3200300741","https://openalex.org/W3217114103","https://openalex.org/W4200027224","https://openalex.org/W4210866342","https://openalex.org/W4220887861","https://openalex.org/W4297510826","https://openalex.org/W4385245566","https://openalex.org/W6749825310","https://openalex.org/W6756353443","https://openalex.org/W6774314701","https://openalex.org/W6780218876","https://openalex.org/W6780576590","https://openalex.org/W6782370274","https://openalex.org/W6784422739","https://openalex.org/W6791278998","https://openalex.org/W6805026025"],"related_works":["https://openalex.org/W4362597605","https://openalex.org/W1574414179","https://openalex.org/W4297676672","https://openalex.org/W3009056573","https://openalex.org/W2922073769","https://openalex.org/W4281702477","https://openalex.org/W2490526372","https://openalex.org/W4376166922","https://openalex.org/W4289763776","https://openalex.org/W2810330923"],"abstract_inverted_index":{"Recently,":[0],"wearable":[1,62],"emotion":[2,63,152,164],"recognition":[3,108,153],"based":[4,39],"on":[5,133,148,183],"peripheral":[6],"physiological":[7,137],"signals":[8],"has":[9],"drawn":[10],"massive":[11],"attention":[12],"due":[13],"to":[14,27,173,180],"its":[15,20],"less":[16],"invasive":[17],"nature":[18],"and":[19,76,84,98,139,177],"applicability":[21],"in":[22,162],"real-life":[23],"scenarios.":[24],"However,":[25],"how":[26],"effectively":[28],"fuse":[29],"multimodal":[30,67,118],"data":[31,89,185],"remains":[32],"a":[33,55,77,110,134],"challenging":[34],"problem.":[35],"Moreover,":[36],"traditional":[37],"fully-supervised":[38,181],"approaches":[40],"suffer":[41],"from":[42],"overfitting":[43],"given":[44],"limited":[45],"labeled":[46],"data.":[47],"To":[48],"address":[49],"the":[50,99,114,126,140,168],"above":[51],"issues,":[52],"we":[53],"propose":[54],"novel":[56],"self-supervised":[57],"learning":[58],"(SSL)":[59],"framework":[60],"for":[61,120],"recognition,":[64],"where":[65],"efficient":[66],"fusion":[68],"is":[69,90,103],"realized":[70],"with":[71,105],"temporal":[72],"convolution-based":[73],"modality-specific":[74],"encoders":[75],"transformer-based":[78],"shared":[79],"encoder,":[80],"capturing":[81],"both":[82],"intra-modal":[83],"inter-modal":[85],"correlations.":[86],"Extensive":[87],"unlabeled":[88],"automatically":[91],"assigned":[92],"labels":[93],"by":[94],"five":[95],"signal":[96,106],"transforms,":[97],"proposed":[100,127,169],"SSL":[101,128],"model":[102,129,170],"pre-trained":[104,132],"transformation":[107],"as":[109],"pretext":[111],"task,":[112],"allowing":[113],"extraction":[115],"of":[116],"generalized":[117],"representations":[119],"emotion-related":[121],"downstream":[122],"tasks.":[123,166],"For":[124],"evaluation,":[125],"was":[130,143,171],"first":[131],"large-scale":[135],"self-collected":[136],"dataset":[138],"resulting":[141],"encoder":[142],"subsequently":[144],"frozen":[145],"or":[146],"fine-tuned":[147],"three":[149],"public":[150],"supervised":[151],"datasets.":[154],"Ultimately,":[155],"our":[156],"SSL-based":[157],"method":[158],"achieved":[159],"state-of-the-art":[160],"results":[161],"various":[163],"classification":[165],"Meanwhile,":[167],"proved":[172],"be":[174],"more":[175],"accurate":[176],"robust":[178],"compared":[179],"methods":[182],"low":[184],"regimes.":[186]},"counts_by_year":[{"year":2026,"cited_by_count":15},{"year":2025,"cited_by_count":47},{"year":2024,"cited_by_count":22},{"year":2023,"cited_by_count":7}],"updated_date":"2026-05-20T08:49:12.498775","created_date":"2025-10-10T00:00:00"}