{"id":"https://openalex.org/W3157840621","doi":"https://doi.org/10.1109/tcyb.2022.3162495","title":"End-to-End Video-to-Speech Synthesis Using Generative Adversarial Networks","display_name":"End-to-End Video-to-Speech Synthesis Using Generative Adversarial Networks","publication_year":2022,"publication_date":"2022-04-19","ids":{"openalex":"https://openalex.org/W3157840621","doi":"https://doi.org/10.1109/tcyb.2022.3162495","mag":"3157840621","pmid":"https://pubmed.ncbi.nlm.nih.gov/35439155"},"language":"en","primary_location":{"id":"doi:10.1109/tcyb.2022.3162495","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcyb.2022.3162495","pdf_url":null,"source":{"id":"https://openalex.org/S4210191041","display_name":"IEEE Transactions on Cybernetics","issn_l":"2168-2267","issn":["2168-2267","2168-2275"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cybernetics","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2104.13332","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013092421","display_name":"Rodrigo Mira","orcid":"https://orcid.org/0000-0002-9493-3842"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Rodrigo Mira","raw_affiliation_strings":["Department of Computing, IBUG Group, Imperial College London, London, U.K"],"raw_orcid":"https://orcid.org/0000-0002-9493-3842","affiliations":[{"raw_affiliation_string":"Department of Computing, IBUG Group, Imperial College London, London, U.K","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079503255","display_name":"Konstantinos Vougioukas","orcid":"https://orcid.org/0000-0001-8552-5559"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Konstantinos Vougioukas","raw_affiliation_strings":["Department of Computing, IBUG Group, Imperial College London, London, U.K"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computing, IBUG Group, Imperial College London, London, U.K","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001790767","display_name":"Pingchuan Ma","orcid":"https://orcid.org/0000-0003-3752-0803"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Pingchuan Ma","raw_affiliation_strings":["Department of Computing, IBUG Group, Imperial College London, London, U.K"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computing, IBUG Group, Imperial College London, London, U.K","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009475700","display_name":"Stavros Petridis","orcid":"https://orcid.org/0000-0001-7478-9479"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Stavros Petridis","raw_affiliation_strings":["Department of Computing, IBUG Group, Imperial College London, London, U.K","FAIAR, Meta AI, London, U.K"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computing, IBUG Group, Imperial College London, London, U.K","institution_ids":["https://openalex.org/I47508984"]},{"raw_affiliation_string":"FAIAR, Meta AI, London, U.K","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043060302","display_name":"Bj\u00f6rn W. Schuller","orcid":"https://orcid.org/0000-0002-6478-8699"},"institutions":[{"id":"https://openalex.org/I179225836","display_name":"University of Augsburg","ror":"https://ror.org/03p14d497","country_code":"DE","type":"education","lineage":["https://openalex.org/I179225836"]},{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["DE","GB"],"is_corresponding":false,"raw_author_name":"Bj\u00f6rn W. Schuller","raw_affiliation_strings":["Department of Computing, IBUG Group, Imperial College London, London, U.K","Chair of Embedded Intelligence for Health Care and Wellbeing, University of Augsburg, Augsburg, Germany"],"raw_orcid":"https://orcid.org/0000-0002-6478-8699","affiliations":[{"raw_affiliation_string":"Department of Computing, IBUG Group, Imperial College London, London, U.K","institution_ids":["https://openalex.org/I47508984"]},{"raw_affiliation_string":"Chair of Embedded Intelligence for Health Care and Wellbeing, University of Augsburg, Augsburg, Germany","institution_ids":["https://openalex.org/I179225836"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016033078","display_name":"Maja Panti\u0107","orcid":"https://orcid.org/0000-0002-3620-5986"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Maja Pantic","raw_affiliation_strings":["Department of Computing, IBUG Group, Imperial College London, London, U.K","FAIAR, Meta AI, London, U.K"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computing, IBUG Group, Imperial College London, London, U.K","institution_ids":["https://openalex.org/I47508984"]},{"raw_affiliation_string":"FAIAR, Meta AI, London, U.K","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.4531,"has_fulltext":false,"cited_by_count":52,"citation_normalized_percentile":{"value":0.97000996,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":99},"biblio":{"volume":"53","issue":"6","first_page":"3454","last_page":"3466"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8260596990585327},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6523857116699219},{"id":"https://openalex.org/keywords/waveform","display_name":"Waveform","score":0.6463577747344971},{"id":"https://openalex.org/keywords/utterance","display_name":"Utterance","score":0.5313779711723328},{"id":"https://openalex.org/keywords/speech-synthesis","display_name":"Speech synthesis","score":0.4665403664112091},{"id":"https://openalex.org/keywords/intelligibility","display_name":"Intelligibility (philosophy)","score":0.4575991928577423},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.39762794971466064},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.08410370349884033}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8260596990585327},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6523857116699219},{"id":"https://openalex.org/C197424946","wikidata":"https://www.wikidata.org/wiki/Q1165717","display_name":"Waveform","level":3,"score":0.6463577747344971},{"id":"https://openalex.org/C2775852435","wikidata":"https://www.wikidata.org/wiki/Q258403","display_name":"Utterance","level":2,"score":0.5313779711723328},{"id":"https://openalex.org/C14999030","wikidata":"https://www.wikidata.org/wiki/Q16346","display_name":"Speech synthesis","level":2,"score":0.4665403664112091},{"id":"https://openalex.org/C60048801","wikidata":"https://www.wikidata.org/wiki/Q1433889","display_name":"Intelligibility (philosophy)","level":2,"score":0.4575991928577423},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39762794971466064},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.08410370349884033},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C554190296","wikidata":"https://www.wikidata.org/wiki/Q47528","display_name":"Radar","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1109/tcyb.2022.3162495","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcyb.2022.3162495","pdf_url":null,"source":{"id":"https://openalex.org/S4210191041","display_name":"IEEE Transactions on Cybernetics","issn_l":"2168-2267","issn":["2168-2267","2168-2275"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cybernetics","raw_type":"journal-article"},{"id":"pmid:35439155","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/35439155","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on cybernetics","raw_type":null},{"id":"pmh:oai:arXiv.org:2104.13332","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2104.13332","pdf_url":"https://arxiv.org/pdf/2104.13332","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:mediatum.ub.tum.de:node/1771612","is_oa":false,"landing_page_url":"https://mediatum.ub.tum.de/1771612","pdf_url":null,"source":{"id":"https://openalex.org/S4377196330","display_name":"mediaTUM  (Technical University of Munich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I62916508","host_organization_name":"Technical University of Munich","host_organization_lineage":["https://openalex.org/I62916508"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"pmh:oai:uni-augsburg.opus-bayern.de:91600","is_oa":false,"landing_page_url":"https://opus.bibliothek.uni-augsburg.de/opus4/frontdoor/index/index/docId/91600","pdf_url":null,"source":{"id":"https://openalex.org/S4306400930","display_name":"OPUS (Augsburg University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I119916105","host_organization_name":"Augsburg University","host_organization_lineage":["https://openalex.org/I119916105"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2104.13332","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2104.13332","pdf_url":"https://arxiv.org/pdf/2104.13332","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.4099999964237213}],"awards":[],"funders":[{"id":"https://openalex.org/F4320332195","display_name":"Samsung","ror":"https://ror.org/04w3jy968"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":85,"referenced_works":["https://openalex.org/W278779762","https://openalex.org/W349236604","https://openalex.org/W1526392145","https://openalex.org/W1538323930","https://openalex.org/W1552314771","https://openalex.org/W1710476689","https://openalex.org/W1995663108","https://openalex.org/W2015143272","https://openalex.org/W2029199293","https://openalex.org/W2035777533","https://openalex.org/W2102737569","https://openalex.org/W2107860279","https://openalex.org/W2115252128","https://openalex.org/W2120847449","https://openalex.org/W2124397839","https://openalex.org/W2125389028","https://openalex.org/W2136155248","https://openalex.org/W2141998673","https://openalex.org/W2159300614","https://openalex.org/W2168744723","https://openalex.org/W2194775991","https://openalex.org/W2290318471","https://openalex.org/W2293856338","https://openalex.org/W2405756170","https://openalex.org/W2471520273","https://openalex.org/W2519091744","https://openalex.org/W2550980560","https://openalex.org/W2585824449","https://openalex.org/W2594690981","https://openalex.org/W2625027024","https://openalex.org/W2769810959","https://openalex.org/W2887437849","https://openalex.org/W2894295011","https://openalex.org/W2910267084","https://openalex.org/W2962879692","https://openalex.org/W2962912639","https://openalex.org/W2963019222","https://openalex.org/W2963066677","https://openalex.org/W2963073614","https://openalex.org/W2963192365","https://openalex.org/W2963528589","https://openalex.org/W2963609956","https://openalex.org/W2963654155","https://openalex.org/W2963782041","https://openalex.org/W2963807156","https://openalex.org/W2963936489","https://openalex.org/W2964243274","https://openalex.org/W2964345931","https://openalex.org/W2964352155","https://openalex.org/W2970006822","https://openalex.org/W2972504708","https://openalex.org/W2972563022","https://openalex.org/W2972756321","https://openalex.org/W2973068670","https://openalex.org/W2973157397","https://openalex.org/W2973215447","https://openalex.org/W2979894294","https://openalex.org/W2981905048","https://openalex.org/W3003241499","https://openalex.org/W3015213852","https://openalex.org/W3015338123","https://openalex.org/W3035626590","https://openalex.org/W3096650361","https://openalex.org/W3101481642","https://openalex.org/W3125708547","https://openalex.org/W4287761884","https://openalex.org/W4294619240","https://openalex.org/W4295521014","https://openalex.org/W4297817572","https://openalex.org/W4298112588","https://openalex.org/W4298289240","https://openalex.org/W4361745739","https://openalex.org/W6632208634","https://openalex.org/W6633117090","https://openalex.org/W6637568146","https://openalex.org/W6677618333","https://openalex.org/W6678815747","https://openalex.org/W6713645886","https://openalex.org/W6732872814","https://openalex.org/W6734491695","https://openalex.org/W6735913928","https://openalex.org/W6755257315","https://openalex.org/W6767111847","https://openalex.org/W6779337556","https://openalex.org/W6779669310"],"related_works":["https://openalex.org/W2529301793","https://openalex.org/W2384121599","https://openalex.org/W2081919107","https://openalex.org/W2127461790","https://openalex.org/W2069324367","https://openalex.org/W1540615732","https://openalex.org/W2039489009","https://openalex.org/W3115948027","https://openalex.org/W2405110677","https://openalex.org/W2811496854"],"abstract_inverted_index":{"Video-to-speech":[0],"is":[1,31,37,101,162,179],"the":[2,6,34,124,128,138,150,154,180,192,202,206,221,231],"process":[3,26],"of":[4,12,87,114,127,140,197,225],"reconstructing":[5],"audio":[7,42,130,152],"speech":[8,166,187],"from":[9,33],"a":[10,13,24,44,47,56,105,109],"video":[11,35,70,94],"spoken":[14,69],"utterance.":[15],"Previous":[16],"approaches":[17],"to":[18,71,104,164,184],"this":[19,52,160],"task":[20],"have":[21],"relied":[22],"on":[23,62,119,241],"two-step":[25],"where":[27],"an":[28,88,115],"intermediate":[29,77],"representation":[30,78],"inferred":[32],"and":[36,97,108,132,153,176,213,223,243],"then":[38,102],"decoded":[39],"into":[40],"waveform":[41,48,72,81,106,131],"using":[43,75],"vocoder":[45],"or":[46,79],"reconstruction":[49],"algorithm.":[50,83],"In":[51,136],"work,":[53],"we":[54],"propose":[55],"new":[57],"end-to-end":[58,73,182],"video-to-speech":[59],"model":[60,85,161,183],"based":[61,118],"generative":[63],"adversarial":[64,116],"networks":[65],"(GANs)":[66],"which":[67,100,219],"translates":[68],"without":[74],"any":[76],"separate":[80],"synthesis":[82,126],"Our":[84],"consists":[86],"encoder-decoder":[89],"architecture":[90],"that":[91,159,177,230],"receives":[92],"raw":[93,129],"as":[95,174],"input":[96,155],"generates":[98],"speech,":[99],"fed":[103],"critic":[107],"power":[110],"critic.":[111],"The":[112],"use":[113,139],"loss":[117],"these":[120],"two":[121,210],"critics":[122],"enables":[123],"direct":[125,147],"ensures":[133],"its":[134],"realism.":[135],"addition,":[137],"our":[141],"three":[142],"comparative":[143],"losses":[144],"helps":[145],"establish":[146],"correspondence":[148],"between":[149],"generated":[151,207],"video.":[156],"We":[157,204,228],"show":[158],"able":[163],"reconstruct":[165],"with":[167],"remarkable":[168],"realism":[169],"for":[170,188],"constrained":[171],"datasets":[172],"such":[173],"GRID,":[175],"it":[178],"first":[181],"produce":[185],"intelligible":[186],"Lip":[189],"Reading":[190],"in":[191,209,238],"Wild":[193],"(LRW),":[194],"featuring":[195],"hundreds":[196],"speakers":[198],"recorded":[199],"entirely":[200],"\"in":[201],"wild.\"":[203],"evaluate":[205],"samples":[208],"different":[211],"scenarios-seen":[212],"unseen":[214],"speakers-using":[215],"four":[216],"objective":[217],"metrics":[218,240],"measure":[220],"quality":[222],"intelligibility":[224],"artificial":[226],"speech.":[227],"demonstrate":[229],"proposed":[232],"approach":[233],"outperforms":[234],"all":[235],"previous":[236],"works":[237],"most":[239],"GRID":[242],"LRW.":[244]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":14},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":2}],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}