{"id":"https://openalex.org/W4393241404","doi":"https://doi.org/10.1109/taslp.2024.3382500","title":"Large-Scale Unsupervised Audio Pre-Training for Video-to-Speech Synthesis","display_name":"Large-Scale Unsupervised Audio Pre-Training for Video-to-Speech Synthesis","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4393241404","doi":"https://doi.org/10.1109/taslp.2024.3382500"},"language":"en","primary_location":{"id":"doi:10.1109/taslp.2024.3382500","is_oa":true,"landing_page_url":"https://doi.org/10.1109/taslp.2024.3382500","pdf_url":"https://ieeexplore.ieee.org/ielx7/6570655/6633080/10480633.pdf","source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://ieeexplore.ieee.org/ielx7/6570655/6633080/10480633.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060152748","display_name":"Triantafyllos Kefalas","orcid":"https://orcid.org/0009-0001-4414-5768"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Triantafyllos Kefalas","raw_affiliation_strings":["Department of Computing, Imperial College London, London, U.K"],"raw_orcid":"https://orcid.org/0009-0001-4414-5768","affiliations":[{"raw_affiliation_string":"Department of Computing, Imperial College London, London, U.K","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050734738","display_name":"Yannis Panagakis","orcid":"https://orcid.org/0000-0003-0153-5210"},"institutions":[{"id":"https://openalex.org/I200777214","display_name":"National and Kapodistrian University of Athens","ror":"https://ror.org/04gnjpq42","country_code":"GR","type":"education","lineage":["https://openalex.org/I200777214"]},{"id":"https://openalex.org/I4210130908","display_name":"Archimedia (Greece)","ror":"https://ror.org/04dxam533","country_code":"GR","type":"company","lineage":["https://openalex.org/I4210130908"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"Yannis Panagakis","raw_affiliation_strings":["Department of Informatics and Telecommunications, National and Kapodistrian University of Athens, Athens, Greece","Archimedes Research Unit, Greece"],"raw_orcid":"https://orcid.org/0000-0003-0153-5210","affiliations":[{"raw_affiliation_string":"Department of Informatics and Telecommunications, National and Kapodistrian University of Athens, Athens, Greece","institution_ids":["https://openalex.org/I200777214"]},{"raw_affiliation_string":"Archimedes Research Unit, Greece","institution_ids":["https://openalex.org/I4210130908"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016033078","display_name":"Maja Panti\u0107","orcid":"https://orcid.org/0000-0002-3620-5986"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Maja Pantic","raw_affiliation_strings":["Department of Computing, Imperial College London, London, U.K"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computing, Imperial College London, London, U.K","institution_ids":["https://openalex.org/I47508984"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.8446,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.68718857,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"32","issue":null,"first_page":"2255","last_page":"2268"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8213129043579102},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6959667801856995},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5891631841659546},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5428036451339722},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5328230261802673},{"id":"https://openalex.org/keywords/spectrogram","display_name":"Spectrogram","score":0.5235754251480103},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.48847997188568115},{"id":"https://openalex.org/keywords/audio-mining","display_name":"Audio mining","score":0.4288172721862793},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3898872137069702},{"id":"https://openalex.org/keywords/voice-activity-detection","display_name":"Voice activity detection","score":0.28642702102661133},{"id":"https://openalex.org/keywords/speech-processing","display_name":"Speech processing","score":0.23863494396209717}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8213129043579102},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6959667801856995},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5891631841659546},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5428036451339722},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5328230261802673},{"id":"https://openalex.org/C45273575","wikidata":"https://www.wikidata.org/wiki/Q578970","display_name":"Spectrogram","level":2,"score":0.5235754251480103},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.48847997188568115},{"id":"https://openalex.org/C157968479","wikidata":"https://www.wikidata.org/wiki/Q3079876","display_name":"Audio mining","level":4,"score":0.4288172721862793},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3898872137069702},{"id":"https://openalex.org/C204201278","wikidata":"https://www.wikidata.org/wiki/Q1332614","display_name":"Voice activity detection","level":3,"score":0.28642702102661133},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.23863494396209717},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taslp.2024.3382500","is_oa":true,"landing_page_url":"https://doi.org/10.1109/taslp.2024.3382500","pdf_url":"https://ieeexplore.ieee.org/ielx7/6570655/6633080/10480633.pdf","source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1109/taslp.2024.3382500","is_oa":true,"landing_page_url":"https://doi.org/10.1109/taslp.2024.3382500","pdf_url":"https://ieeexplore.ieee.org/ielx7/6570655/6633080/10480633.pdf","source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7064714105","display_name":"DTP 2016-2017 Imperal College","funder_award_id":"EP/N509486/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G708676777","display_name":"DTP - Imperial College London","funder_award_id":"EP/M507878/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393241404.pdf","grobid_xml":"https://content.openalex.org/works/W4393241404.grobid-xml"},"referenced_works_count":103,"referenced_works":["https://openalex.org/W1494198834","https://openalex.org/W1552314771","https://openalex.org/W1924770834","https://openalex.org/W2015143272","https://openalex.org/W2029199293","https://openalex.org/W2067295501","https://openalex.org/W2078711706","https://openalex.org/W2079735306","https://openalex.org/W2106284211","https://openalex.org/W2120847449","https://openalex.org/W2124397839","https://openalex.org/W2127141656","https://openalex.org/W2136155248","https://openalex.org/W2148154194","https://openalex.org/W2152826865","https://openalex.org/W2194775991","https://openalex.org/W2243738093","https://openalex.org/W2293856338","https://openalex.org/W2325939864","https://openalex.org/W2398406965","https://openalex.org/W2404704342","https://openalex.org/W2509287561","https://openalex.org/W2516001803","https://openalex.org/W2516412152","https://openalex.org/W2556171197","https://openalex.org/W2585824449","https://openalex.org/W2593414223","https://openalex.org/W2594690981","https://openalex.org/W2625027024","https://openalex.org/W2726515241","https://openalex.org/W2782717521","https://openalex.org/W2808631503","https://openalex.org/W2890952074","https://openalex.org/W2891205112","https://openalex.org/W2949662773","https://openalex.org/W2963019222","https://openalex.org/W2963173190","https://openalex.org/W2963192365","https://openalex.org/W2963307811","https://openalex.org/W2963528589","https://openalex.org/W2963654155","https://openalex.org/W2964171275","https://openalex.org/W2964243274","https://openalex.org/W2964352155","https://openalex.org/W2972359262","https://openalex.org/W2972563022","https://openalex.org/W2972756321","https://openalex.org/W2973157397","https://openalex.org/W2998572311","https://openalex.org/W3003241499","https://openalex.org/W3011424113","https://openalex.org/W3015338123","https://openalex.org/W3035626590","https://openalex.org/W3096650361","https://openalex.org/W3097777922","https://openalex.org/W3104792420","https://openalex.org/W3131384824","https://openalex.org/W3136499730","https://openalex.org/W3144035034","https://openalex.org/W3157840621","https://openalex.org/W3160305627","https://openalex.org/W3161236344","https://openalex.org/W3162293946","https://openalex.org/W3164223074","https://openalex.org/W3188692592","https://openalex.org/W3198020407","https://openalex.org/W3198694222","https://openalex.org/W3213322812","https://openalex.org/W4200635083","https://openalex.org/W4206204999","https://openalex.org/W4221143458","https://openalex.org/W4282946496","https://openalex.org/W4292773472","https://openalex.org/W4296069328","https://openalex.org/W4298112588","https://openalex.org/W4306246947","https://openalex.org/W4307286264","https://openalex.org/W4312975157","https://openalex.org/W4390872449","https://openalex.org/W6631190155","https://openalex.org/W6638667902","https://openalex.org/W6640212811","https://openalex.org/W6640963894","https://openalex.org/W6652300196","https://openalex.org/W6679045638","https://openalex.org/W6695676441","https://openalex.org/W6726497184","https://openalex.org/W6732872814","https://openalex.org/W6741832134","https://openalex.org/W6752888775","https://openalex.org/W6754420807","https://openalex.org/W6757817989","https://openalex.org/W6762533536","https://openalex.org/W6767111847","https://openalex.org/W6771467084","https://openalex.org/W6772349387","https://openalex.org/W6779192484","https://openalex.org/W6779823529","https://openalex.org/W6783713337","https://openalex.org/W6783867762","https://openalex.org/W6787647647","https://openalex.org/W6795747053","https://openalex.org/W6803359641"],"related_works":["https://openalex.org/W2530685530","https://openalex.org/W4375868962","https://openalex.org/W2011227383","https://openalex.org/W2088854863","https://openalex.org/W4402568167","https://openalex.org/W3179495260","https://openalex.org/W1976719989","https://openalex.org/W3127543252","https://openalex.org/W2065606036","https://openalex.org/W2016904525"],"abstract_inverted_index":{"Video-to-speech":[0],"synthesis":[1,94],"is":[2,127],"the":[3,7,38,83,88,92,121,133,136,149],"task":[4,141],"of":[5,14,40,75,135,148],"reconstructing":[6],"speech":[8,58,123],"signal":[9],"from":[10,22,111,146],"a":[11,15,32,48,139],"silent":[12],"video":[13,34],"speaker.":[16],"Previous":[17],"approaches":[18],"train":[19,67],"on":[20,70],"data":[21,77],"almost":[23],"exclusively":[24],"audio-visual":[25],"datasets,":[26],"i.e.,":[27],"every":[28],"audio":[29,76,89,100,157],"sample":[30],"has":[31],"corresponding":[33,49,109],"sample.":[35],"This":[36],"precludes":[37],"use":[39,82],"abundant":[41],"audio-only":[42],"datasets":[43],"which":[44],"may":[45],"not":[46,105],"have":[47],"visual":[50],"modality":[51],"such":[52],"as":[53,161],"audiobooks,":[54],"radio":[55],"podcasts,":[56],"and":[57,80,103,124,158,164],"recognition":[59],"datasets.":[60],"In":[61],"this":[62,119],"paper":[63],"we":[64],"propose":[65],"to":[66,86,131],"encoder-decoder":[68],"models":[69,167],"more":[71],"than":[72],"3,500":[73],"hours":[74],"at":[78],"24kHz,":[79],"then":[81],"pretrained":[84],"decoders":[85,90],"initialize":[87],"for":[91],"video-tospeech":[93],"task.":[95],"The":[96],"pre-training":[97],"step":[98],"uses":[99],"samples":[101,110,145],"only":[102,143],"does":[104],"require":[106],"labels":[107],"or":[108],"other":[112],"modalities":[113],"(visual,":[114],"text).":[115],"We":[116,151],"demonstrate":[117],"that":[118,125],"improves":[120],"reconstructed":[122],"it":[126],"an":[128],"unexplored":[129],"way":[130],"improve":[132],"quality":[134],"generator":[137],"in":[138],"cross-modal":[140],"while":[142],"requiring":[144],"one":[147],"modalities.":[150],"conduct":[152],"experiments":[153],"using":[154],"both":[155],"raw":[156],"mel":[159],"spectrograms":[160],"target":[162],"outputs":[163],"benchmark":[165],"our":[166],"with":[168],"existing":[169],"work.":[170]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}