{"id":"https://openalex.org/W3083394627","doi":"https://doi.org/10.1109/taffc.2020.3022017","title":"Learning Continuous Facial Actions From Speech for Real-Time Animation","display_name":"Learning Continuous Facial Actions From Speech for Real-Time Animation","publication_year":2020,"publication_date":"2020-09-04","ids":{"openalex":"https://openalex.org/W3083394627","doi":"https://doi.org/10.1109/taffc.2020.3022017","mag":"3083394627"},"language":"en","primary_location":{"id":"doi:10.1109/taffc.2020.3022017","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2020.3022017","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5017287333","display_name":"Hai Pham","orcid":"https://orcid.org/0000-0001-9391-5514"},"institutions":[{"id":"https://openalex.org/I4210117523","display_name":"Samsung (United Kingdom)","ror":"https://ror.org/01w6gjq94","country_code":"GB","type":"company","lineage":["https://openalex.org/I2250650973","https://openalex.org/I4210117523"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Hai X. Pham","raw_affiliation_strings":["Samsung AI Center, Cambridge, U.K"],"raw_orcid":"https://orcid.org/0000-0001-9391-5514","affiliations":[{"raw_affiliation_string":"Samsung AI Center, Cambridge, U.K","institution_ids":["https://openalex.org/I4210117523"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100384201","display_name":"Yuting Wang","orcid":"https://orcid.org/0000-0002-1562-2764"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuting Wang","raw_affiliation_strings":["Department of Computer Science, Rutgers University, Piscataway, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020368096","display_name":"Vladimir Pavlovi\u0107","orcid":"https://orcid.org/0000-0003-3979-1236"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vladimir Pavlovic","raw_affiliation_strings":["Department of Computer Science, Rutgers University, Piscataway, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.1917,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.49120382,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"13","issue":"3","first_page":"1567","last_page":"1580"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/facial-expression","display_name":"Facial expression","score":0.7633306384086609},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7444460988044739},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6814628839492798},{"id":"https://openalex.org/keywords/computer-facial-animation","display_name":"Computer facial animation","score":0.6593762636184692},{"id":"https://openalex.org/keywords/spectrogram","display_name":"Spectrogram","score":0.5933021306991577},{"id":"https://openalex.org/keywords/utterance","display_name":"Utterance","score":0.5467339158058167},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.5243837237358093},{"id":"https://openalex.org/keywords/facial-motion-capture","display_name":"Facial motion capture","score":0.5192944407463074},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.5118278861045837},{"id":"https://openalex.org/keywords/animation","display_name":"Animation","score":0.5042554140090942},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.48178479075431824},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.47813522815704346},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.45037826895713806},{"id":"https://openalex.org/keywords/face","display_name":"Face (sociological concept)","score":0.4500397741794586},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.4305487871170044},{"id":"https://openalex.org/keywords/computer-animation","display_name":"Computer animation","score":0.335578978061676},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.2934401035308838},{"id":"https://openalex.org/keywords/facial-recognition-system","display_name":"Facial recognition system","score":0.22512251138687134},{"id":"https://openalex.org/keywords/face-detection","display_name":"Face detection","score":0.14760258793830872}],"concepts":[{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.7633306384086609},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7444460988044739},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6814628839492798},{"id":"https://openalex.org/C138591656","wikidata":"https://www.wikidata.org/wiki/Q5157538","display_name":"Computer facial animation","level":4,"score":0.6593762636184692},{"id":"https://openalex.org/C45273575","wikidata":"https://www.wikidata.org/wiki/Q578970","display_name":"Spectrogram","level":2,"score":0.5933021306991577},{"id":"https://openalex.org/C2775852435","wikidata":"https://www.wikidata.org/wiki/Q258403","display_name":"Utterance","level":2,"score":0.5467339158058167},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5243837237358093},{"id":"https://openalex.org/C98907195","wikidata":"https://www.wikidata.org/wiki/Q5428562","display_name":"Facial motion capture","level":5,"score":0.5192944407463074},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.5118278861045837},{"id":"https://openalex.org/C502989409","wikidata":"https://www.wikidata.org/wiki/Q11425","display_name":"Animation","level":2,"score":0.5042554140090942},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48178479075431824},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.47813522815704346},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.45037826895713806},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.4500397741794586},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.4305487871170044},{"id":"https://openalex.org/C69369342","wikidata":"https://www.wikidata.org/wiki/Q1401416","display_name":"Computer animation","level":3,"score":0.335578978061676},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.2934401035308838},{"id":"https://openalex.org/C31510193","wikidata":"https://www.wikidata.org/wiki/Q1192553","display_name":"Facial recognition system","level":3,"score":0.22512251138687134},{"id":"https://openalex.org/C4641261","wikidata":"https://www.wikidata.org/wiki/Q11681085","display_name":"Face detection","level":4,"score":0.14760258793830872},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taffc.2020.3022017","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2020.3022017","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5799999833106995,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W34110307","https://openalex.org/W71477782","https://openalex.org/W219197677","https://openalex.org/W1482998036","https://openalex.org/W1538131130","https://openalex.org/W1542280630","https://openalex.org/W1600744878","https://openalex.org/W1779294284","https://openalex.org/W1924770834","https://openalex.org/W1995562189","https://openalex.org/W2003238582","https://openalex.org/W2015143272","https://openalex.org/W2017107803","https://openalex.org/W2033310064","https://openalex.org/W2036242736","https://openalex.org/W2043440654","https://openalex.org/W2064400654","https://openalex.org/W2064675550","https://openalex.org/W2090777335","https://openalex.org/W2102003408","https://openalex.org/W2107813907","https://openalex.org/W2114336453","https://openalex.org/W2134973740","https://openalex.org/W2142487393","https://openalex.org/W2144005487","https://openalex.org/W2147885303","https://openalex.org/W2150133190","https://openalex.org/W2155273149","https://openalex.org/W2163334800","https://openalex.org/W2239141610","https://openalex.org/W2296650210","https://openalex.org/W2329093554","https://openalex.org/W2338415399","https://openalex.org/W2398826216","https://openalex.org/W2399733683","https://openalex.org/W2406901294","https://openalex.org/W2513507089","https://openalex.org/W2566655602","https://openalex.org/W2608566907","https://openalex.org/W2735227950","https://openalex.org/W2737658251","https://openalex.org/W2738406145","https://openalex.org/W2739192055","https://openalex.org/W2745771616","https://openalex.org/W2803193013","https://openalex.org/W2895226286","https://openalex.org/W2963175699","https://openalex.org/W2963503541","https://openalex.org/W2979894294","https://openalex.org/W3006295631","https://openalex.org/W4236533540","https://openalex.org/W4385490328","https://openalex.org/W6629329631","https://openalex.org/W6632100814","https://openalex.org/W6640212811","https://openalex.org/W6676946368","https://openalex.org/W6737896281","https://openalex.org/W6741016902"],"related_works":["https://openalex.org/W1544039745","https://openalex.org/W2748502319","https://openalex.org/W2401874398","https://openalex.org/W2906438086","https://openalex.org/W615219140","https://openalex.org/W2165663879","https://openalex.org/W4385825592","https://openalex.org/W2188148675","https://openalex.org/W2096968018","https://openalex.org/W3040900128"],"abstract_inverted_index":{"Speech":[0],"conveys":[1],"not":[2,66],"only":[3,67],"the":[4,15,35,80,90,105,122,158],"verbal":[5],"communication,":[6],"but":[7,95],"also":[8],"emotions,":[9],"manifested":[10],"as":[11,75],"facial":[12,28,45,70,118,133,161,177],"expressions":[13,29],"of":[14,92,104,116,160,169,176,185],"speaker.":[16],"In":[17,121],"this":[18],"article,":[19],"we":[20,136],"present":[21],"deep":[22],"learning":[23],"frameworks":[24],"that":[25,138],"directly":[26],"infer":[27],"from":[30,149],"just":[31],"speech":[32,150],"signals.":[33],"Specifically,":[34],"time-varying":[36],"contextual":[37],"non-linear":[38],"mapping":[39],"between":[40],"audio":[41],"stream":[42],"and":[43,107,179,202],"micro":[44],"movements":[46],"is":[47,140],"realized":[48],"by":[49,113],"our":[50,186,190],"proposed":[51],"recurrent":[52],"neural":[53],"networks":[54],"to":[55,83,131,143,196],"drive":[56],"a":[57,173],"3D":[58,76],"blendshape":[59],"face":[60],"model":[61],"in":[62,79,89,199],"real-time.":[63],"Our":[64],"models":[65,192],"activate":[68],"appropriate":[69],"action":[71,162],"units":[72],"(AUs),":[73],"defined":[74],"expression":[77],"blendshapes":[78],"FaceWarehouse":[81],"database,":[82],"depict":[84],"different":[85,170],"utterance":[86],"generating":[87],"actions":[88,178],"form":[91],"lip":[93],"movements,":[94],"also,":[96],"without":[97],"any":[98],"assumption,":[99],"automatically":[100],"estimate":[101],"emotional":[102,180],"intensity":[103],"speaker":[106],"reproduces":[108],"her":[109],"ever-changing":[110],"affective":[111],"states":[112,181],"adjusting":[114],"strength":[115],"related":[117],"unit":[119],"activations.":[120],"baseline":[123],"models,":[124],"conventional":[125],"handcrafted":[126],"acoustic":[127,146],"features":[128],"are":[129,193],"utilized":[130],"predict":[132],"actions.":[134],"Furthermore,":[135],"show":[137,182],"it":[139],"more":[141],"advantageous":[142],"learn":[144],"meaningful":[145],"feature":[147],"representation":[148],"spectrograms":[151],"with":[152],"convolutional":[153],"nets,":[154],"which":[155],"subsequently":[156],"improves":[157],"accuracy":[159],"synthesis.":[163],"Experiments":[164],"on":[165],"diverse":[166],"audiovisual":[167],"corpora":[168],"actors":[171],"across":[172],"wide":[174],"range":[175],"promising":[183],"results":[184],"approaches.":[187],"Being":[188],"speaker-independent,":[189],"generalized":[191],"readily":[194],"applicable":[195],"various":[197],"tasks":[198],"human-machine":[200],"interaction":[201],"animation.":[203]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}