{"id":"https://openalex.org/W3098994423","doi":"https://doi.org/10.1109/taffc.2021.3062406","title":"Does Visual Self-Supervision Improve Learning of Speech Representations for Emotion Recognition?","display_name":"Does Visual Self-Supervision Improve Learning of Speech Representations for Emotion Recognition?","publication_year":2021,"publication_date":"2021-02-26","ids":{"openalex":"https://openalex.org/W3098994423","doi":"https://doi.org/10.1109/taffc.2021.3062406","mag":"3098994423"},"language":"en","primary_location":{"id":"doi:10.1109/taffc.2021.3062406","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2021.3062406","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2005.01400","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Abhinav Shukla","orcid":"https://orcid.org/0000-0003-4030-962X"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Abhinav Shukla","raw_affiliation_strings":["iBUG Group, Department of Computing, Imperial College London, London, U.K"],"raw_orcid":"https://orcid.org/0000-0003-4030-962X","affiliations":[{"raw_affiliation_string":"iBUG Group, Department of Computing, Imperial College London, London, U.K","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Stavros Petridis","orcid":null},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Stavros Petridis","raw_affiliation_strings":["iBUG Group, Department of Computing, Imperial College London, London, U.K"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"iBUG Group, Department of Computing, Imperial College London, London, U.K","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"last","author":{"id":null,"display_name":"Maja Pantic","orcid":null},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Maja Pantic","raw_affiliation_strings":["iBUG Group, Department of Computing, Imperial College London, London, U.K"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"iBUG Group, Department of Computing, Imperial College London, London, U.K","institution_ids":["https://openalex.org/I47508984"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I47508984"],"apc_list":null,"apc_paid":null,"fwci":3.5209,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.92673342,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"14","issue":"1","first_page":"406","last_page":"420"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9269000291824341,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9269000291824341,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.014100000262260437,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.011599999852478504,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/overfitting","display_name":"Overfitting","score":0.7174000144004822},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.5461999773979187},{"id":"https://openalex.org/keywords/audio-visual","display_name":"Audio visual","score":0.49079999327659607},{"id":"https://openalex.org/keywords/feature-learning","display_name":"Feature learning","score":0.4855000078678131},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.46480000019073486},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4235000014305115},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.42329999804496765},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.41190001368522644},{"id":"https://openalex.org/keywords/face","display_name":"Face (sociological concept)","score":0.3499000072479248}],"concepts":[{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.7174000144004822},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6764000058174133},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6233000159263611},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.5461999773979187},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49970000982284546},{"id":"https://openalex.org/C3017588708","wikidata":"https://www.wikidata.org/wiki/Q758901","display_name":"Audio visual","level":2,"score":0.49079999327659607},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.4855000078678131},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.46480000019073486},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4235000014305115},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.42329999804496765},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.41190001368522644},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.3499000072479248},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.311599999666214},{"id":"https://openalex.org/C28006648","wikidata":"https://www.wikidata.org/wiki/Q6934509","display_name":"Multi-task learning","level":3,"score":0.305400013923645},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.3027999997138977},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2994000017642975},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.29760000109672546},{"id":"https://openalex.org/C6438553","wikidata":"https://www.wikidata.org/wiki/Q1185804","display_name":"Affective computing","level":2,"score":0.2858000099658966},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.27250000834465027},{"id":"https://openalex.org/C2779321571","wikidata":"https://www.wikidata.org/wiki/Q7936605","display_name":"Visual learning","level":2,"score":0.27160000801086426},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.27129998803138733},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.2685000002384186},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.266400009393692},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.26489999890327454},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2612999975681305},{"id":"https://openalex.org/C206310091","wikidata":"https://www.wikidata.org/wiki/Q750859","display_name":"Emotion classification","level":2,"score":0.25690001249313354},{"id":"https://openalex.org/C133892786","wikidata":"https://www.wikidata.org/wiki/Q1145189","display_name":"Speaker recognition","level":2,"score":0.2556999921798706}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/taffc.2021.3062406","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2021.3062406","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2005.01400","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2005.01400","pdf_url":"https://arxiv.org/pdf/2005.01400","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2005.01400","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2005.01400","pdf_url":"https://arxiv.org/pdf/2005.01400","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320332195","display_name":"Samsung","ror":"https://ror.org/04w3jy968"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":67,"referenced_works":["https://openalex.org/W343636949","https://openalex.org/W1901129140","https://openalex.org/W1974387177","https://openalex.org/W2015394094","https://openalex.org/W2030931454","https://openalex.org/W2045528981","https://openalex.org/W2090777335","https://openalex.org/W2146334809","https://openalex.org/W2147885303","https://openalex.org/W2163928333","https://openalex.org/W2261706002","https://openalex.org/W2550462002","https://openalex.org/W2594690981","https://openalex.org/W2738406145","https://openalex.org/W2751861701","https://openalex.org/W2753927079","https://openalex.org/W2794209590","https://openalex.org/W2798536775","https://openalex.org/W2803193013","https://openalex.org/W2883409523","https://openalex.org/W2883725317","https://openalex.org/W2910165986","https://openalex.org/W2913939497","https://openalex.org/W2962739339","https://openalex.org/W2962931510","https://openalex.org/W2962960500","https://openalex.org/W2962969419","https://openalex.org/W2963115079","https://openalex.org/W2963654155","https://openalex.org/W2963702064","https://openalex.org/W2964700958","https://openalex.org/W2972563022","https://openalex.org/W2972894903","https://openalex.org/W2972943112","https://openalex.org/W2973049979","https://openalex.org/W2973157397","https://openalex.org/W2979894294","https://openalex.org/W2981952041","https://openalex.org/W2982039329","https://openalex.org/W2999863388","https://openalex.org/W3006295631","https://openalex.org/W3015356123","https://openalex.org/W3016081131","https://openalex.org/W3016181583","https://openalex.org/W3034781633","https://openalex.org/W3035118106","https://openalex.org/W3035160371","https://openalex.org/W3035524453","https://openalex.org/W3100270690","https://openalex.org/W3108655343","https://openalex.org/W3154807520","https://openalex.org/W3175300676","https://openalex.org/W4289440497","https://openalex.org/W4289665794","https://openalex.org/W6747899497","https://openalex.org/W6750665317","https://openalex.org/W6751750676","https://openalex.org/W6754048563","https://openalex.org/W6756859573","https://openalex.org/W6765052341","https://openalex.org/W6770805772","https://openalex.org/W6775244068","https://openalex.org/W6779997284","https://openalex.org/W6780637495","https://openalex.org/W6783915111","https://openalex.org/W6840487619","https://openalex.org/W6844194202"],"related_works":[],"abstract_inverted_index":{"Self-supervised":[0],"learning":[1,49,79,148],"has":[2,21],"attracted":[3],"plenty":[4],"of":[5,50,69,142],"recent":[6],"research":[7],"interest.":[8],"However,":[9],"most":[10],"works":[11],"for":[12,34,59,78,116,132,145,163],"self-supervision":[13,42,57,75,144,156],"in":[14,86],"speech":[15,60,125,164],"are":[16,83],"typically":[17],"unimodal":[18],"and":[19,31,73,99,123,149,154,165],"there":[20],"been":[22],"limited":[23],"work":[24],"that":[25,65,82,91,151],"studies":[26],"the":[27,48,70,140],"interaction":[28],"between":[29],"audio":[30,51,74,114,146,155,161],"visual":[32,41,72,143,153],"modalities":[33],"cross-modal":[35],"self-supervision.":[36],"This":[37],"article":[38],"(1)":[39],"investigates":[40],"via":[43],"face":[44],"reconstruction":[45],"to":[46,103,158],"guide":[47],"representations;":[52],"(2)":[53],"proposes":[54],"an":[55],"audio-only":[56],"approach":[58],"representation":[61],"learning;":[62],"(3)":[63],"shows":[64,90],"a":[66],"multi-task":[67],"combination":[68],"proposed":[71],"is":[76,100],"beneficial":[77],"richer":[80],"features":[81],"more":[84,159],"robust":[85],"noisy":[87],"conditions;":[88],"(4)":[89],"self-supervised":[92,130],"pretraining":[93],"can":[94],"outperform":[95,128],"fully":[96],"supervised":[97],"training":[98],"especially":[101],"useful":[102],"prevent":[104],"overfitting":[105],"on":[106],"smaller":[107],"sized":[108],"datasets.":[109],"We":[110,127],"evaluate":[111],"our":[112],"learned":[113],"representations":[115,162],"discrete":[117],"emotion":[118,166],"recognition,":[119],"continuous":[120],"affect":[121],"recognition":[122],"automatic":[124],"recognition.":[126,167],"existing":[129],"methods":[131],"all":[133],"tested":[134],"downstream":[135],"tasks.":[136],"Our":[137],"results":[138],"demonstrate":[139],"potential":[141],"feature":[147],"suggest":[150],"joint":[152],"leads":[157],"informative":[160]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":7}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2020-11-23T00:00:00"}