{"id":"https://openalex.org/W3029296930","doi":"https://doi.org/10.1145/3382507.3418840","title":"Modality Dropout for Improved Performance-driven Talking Faces","display_name":"Modality Dropout for Improved Performance-driven Talking Faces","publication_year":2020,"publication_date":"2020-10-21","ids":{"openalex":"https://openalex.org/W3029296930","doi":"https://doi.org/10.1145/3382507.3418840","mag":"3029296930"},"language":"en","primary_location":{"id":"doi:10.1145/3382507.3418840","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3382507.3418840","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 International Conference on Multimodal Interaction","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2005.13616","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101807751","display_name":"Ahmed Hussen Abdelaziz","orcid":"https://orcid.org/0000-0001-8027-4666"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ahmed Hussen Abdelaziz","raw_affiliation_strings":["Apple, Cupertino, CA, USA","Apple, Cupertino, CA, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]},{"raw_affiliation_string":"Apple, Cupertino, CA, USA#TAB#","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112911728","display_name":"Barry-John Theobald","orcid":null},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Barry-John Theobald","raw_affiliation_strings":["Apple, Cupertino, CA, USA","Apple, Cupertino, CA, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]},{"raw_affiliation_string":"Apple, Cupertino, CA, USA#TAB#","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049896100","display_name":"Paul R. Dixon","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paul Dixon","raw_affiliation_strings":["Apple, Zurick, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, Zurick, Switzerland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035971994","display_name":"Reinhard Knothe","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Reinhard Knothe","raw_affiliation_strings":["Apple, Zurick, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, Zurick, Switzerland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071541814","display_name":"Nicholas Apostoloff","orcid":null},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nicholas Apostoloff","raw_affiliation_strings":["Apple, Cupertino, CA, USA","Apple, Cupertino, CA, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]},{"raw_affiliation_string":"Apple, Cupertino, CA, USA#TAB#","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091124471","display_name":"Sachin Kajareker","orcid":null},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sachin Kajareker","raw_affiliation_strings":["Apple, Cupertino, CA, USA","Apple, Cupertino, CA, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]},{"raw_affiliation_string":"Apple, Cupertino, CA, USA#TAB#","institution_ids":["https://openalex.org/I4210153776"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"378","last_page":"386"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10531","display_name":"Advanced Vision and Imaging","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.819452166557312},{"id":"https://openalex.org/keywords/dropout","display_name":"Dropout (neural networks)","score":0.6698184013366699},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.6383090019226074},{"id":"https://openalex.org/keywords/animation","display_name":"Animation","score":0.6377065777778625},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.5255818963050842},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.4998769760131836},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.49917054176330566},{"id":"https://openalex.org/keywords/computer-facial-animation","display_name":"Computer facial animation","score":0.490116685628891},{"id":"https://openalex.org/keywords/phone","display_name":"Phone","score":0.46928274631500244},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4001130759716034},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.3605983257293701},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3410130739212036},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3321409225463867},{"id":"https://openalex.org/keywords/computer-animation","display_name":"Computer animation","score":0.29531148076057434},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.14743635058403015},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.13187429308891296}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.819452166557312},{"id":"https://openalex.org/C2776145597","wikidata":"https://www.wikidata.org/wiki/Q25339462","display_name":"Dropout (neural networks)","level":2,"score":0.6698184013366699},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.6383090019226074},{"id":"https://openalex.org/C502989409","wikidata":"https://www.wikidata.org/wiki/Q11425","display_name":"Animation","level":2,"score":0.6377065777778625},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.5255818963050842},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.4998769760131836},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.49917054176330566},{"id":"https://openalex.org/C138591656","wikidata":"https://www.wikidata.org/wiki/Q5157538","display_name":"Computer facial animation","level":4,"score":0.490116685628891},{"id":"https://openalex.org/C2778707766","wikidata":"https://www.wikidata.org/wiki/Q202064","display_name":"Phone","level":2,"score":0.46928274631500244},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4001130759716034},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3605983257293701},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3410130739212036},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3321409225463867},{"id":"https://openalex.org/C69369342","wikidata":"https://www.wikidata.org/wiki/Q1401416","display_name":"Computer animation","level":3,"score":0.29531148076057434},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.14743635058403015},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.13187429308891296},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3382507.3418840","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3382507.3418840","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 International Conference on Multimodal Interaction","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2005.13616","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2005.13616","pdf_url":"https://arxiv.org/pdf/2005.13616","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"doi:10.48550/arxiv.2005.13616","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2005.13616","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:3029296930","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2005.13616","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2005.13616","pdf_url":"https://arxiv.org/pdf/2005.13616","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7300000190734863}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3029296930.pdf","grobid_xml":"https://content.openalex.org/works/W3029296930.grobid-xml"},"referenced_works_count":66,"referenced_works":["https://openalex.org/W63958146","https://openalex.org/W132898664","https://openalex.org/W1569907127","https://openalex.org/W1588539311","https://openalex.org/W1778065289","https://openalex.org/W1973757125","https://openalex.org/W1974357398","https://openalex.org/W1976295221","https://openalex.org/W1987875999","https://openalex.org/W1992187125","https://openalex.org/W2000911139","https://openalex.org/W2004789217","https://openalex.org/W2012048984","https://openalex.org/W2024490110","https://openalex.org/W2062712751","https://openalex.org/W2063634999","https://openalex.org/W2069997605","https://openalex.org/W2070726616","https://openalex.org/W2086331119","https://openalex.org/W2102416463","https://openalex.org/W2110501904","https://openalex.org/W2114336453","https://openalex.org/W2120067677","https://openalex.org/W2120654454","https://openalex.org/W2128173845","https://openalex.org/W2129360799","https://openalex.org/W2146991130","https://openalex.org/W2147885303","https://openalex.org/W2162598851","https://openalex.org/W2163685610","https://openalex.org/W2180648442","https://openalex.org/W2289286917","https://openalex.org/W2295661697","https://openalex.org/W2474891805","https://openalex.org/W2515372520","https://openalex.org/W2623464795","https://openalex.org/W2721682741","https://openalex.org/W2725151104","https://openalex.org/W2737658251","https://openalex.org/W2738406145","https://openalex.org/W2738581557","https://openalex.org/W2739192055","https://openalex.org/W2762899171","https://openalex.org/W2774812352","https://openalex.org/W2780113219","https://openalex.org/W2790649793","https://openalex.org/W2796931171","https://openalex.org/W2797032258","https://openalex.org/W2804600264","https://openalex.org/W2806723288","https://openalex.org/W2890952074","https://openalex.org/W2895226286","https://openalex.org/W2902299888","https://openalex.org/W2906447902","https://openalex.org/W2944661720","https://openalex.org/W2950864153","https://openalex.org/W2951529591","https://openalex.org/W2952746495","https://openalex.org/W2963192057","https://openalex.org/W2964559396","https://openalex.org/W2980562267","https://openalex.org/W2995034616","https://openalex.org/W3006295631","https://openalex.org/W4238101556","https://openalex.org/W6644369777","https://openalex.org/W6757270357"],"related_works":["https://openalex.org/W2079723135","https://openalex.org/W3114960734","https://openalex.org/W2129138066","https://openalex.org/W3081461879","https://openalex.org/W2981822321","https://openalex.org/W2153546194","https://openalex.org/W1516978277","https://openalex.org/W1815577581","https://openalex.org/W2398288157","https://openalex.org/W2053099149","https://openalex.org/W3214083601","https://openalex.org/W2044700703","https://openalex.org/W2083792893","https://openalex.org/W108030751","https://openalex.org/W2111649735","https://openalex.org/W2134961334","https://openalex.org/W1539720309","https://openalex.org/W2123795913","https://openalex.org/W2044254905","https://openalex.org/W3181255780"],"abstract_inverted_index":{"We":[0,111],"describe":[1],"our":[2,40],"novel":[3],"deep":[4],"learning":[5],"approach":[6],"for":[7,160,167,177],"driving":[8],"animated":[9],"faces":[10],"using":[11,24,33],"both":[12,43],"acoustic":[13],"and":[14,27,54,75,98,128],"visual":[15,35,76],"information.":[16,36],"In":[17],"particular,":[18],"speech-related":[19,136],"facial":[20,29,137],"movements":[21,30,138],"are":[22,31,48],"generated":[23,32,49],"audiovisual":[25,55],"information,":[26],"non-verbal":[28],"only":[34,158],"To":[37],"ensure":[38],"that":[39,50],"model":[41,72,82],"exploits":[42,73],"modalities":[44],"during":[45,78],"training,":[46],"batches":[47],"contain":[51],"audio-only,":[52],"video-only,":[53],"input":[56],"features.":[57],"The":[58],"probability":[59],"of":[60,108,120,135,152],"dropping":[61],"a":[62,91,104],"modality":[63,141,144],"allows":[64],"control":[65],"over":[66,123],"the":[67,71,109,118,124,130,133,153],"degree":[68],"to":[69,115,171,175],"which":[70],"audio":[74],"information":[77],"training.":[79],"Our":[80],"trained":[81],"runs":[83],"in":[84,132,150],"real-time":[85],"on":[86,103],"resource":[87],"limited":[88],"hardware":[89],"(e.g.":[90],"smart":[92],"phone),":[93],"it":[94,99],"is":[95,100],"user":[96],"agnostic,":[97],"not":[101],"dependent":[102],"potentially":[105],"error-prone":[106],"transcription":[107],"speech.":[110],"use":[112],"subjective":[113],"testing":[114],"demonstrate:":[116],"1)":[117],"improvement":[119,131],"audiovisual-driven":[121,148,168],"animation":[122,134,149,169],"equivalent":[125],"video-only":[126],"approach,":[127],"2)":[129],"after":[139],"introducing":[140,163],"dropout.":[142],"Without":[143],"dropout,":[145],"viewers":[146],"prefer":[147],"51%":[151],"test":[154],"sequences":[155],"compared":[156],"with":[157],"18%":[159],"video-driven.":[161],"After":[162],"dropout":[164],"viewer":[165],"preference":[166],"increases":[170],"74%,":[172],"but":[173],"decreases":[174],"8%":[176],"video-only.":[178]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}