{"id":"https://openalex.org/W4405967193","doi":"https://doi.org/10.3390/computers14010007","title":"Seeing the Sound: Multilingual Lip Sync for Real-Time Face-to-Face Translation","display_name":"Seeing the Sound: Multilingual Lip Sync for Real-Time Face-to-Face Translation","publication_year":2024,"publication_date":"2024-12-28","ids":{"openalex":"https://openalex.org/W4405967193","doi":"https://doi.org/10.3390/computers14010007"},"language":"en","primary_location":{"id":"doi:10.3390/computers14010007","is_oa":true,"landing_page_url":"https://doi.org/10.3390/computers14010007","pdf_url":"https://www.mdpi.com/2073-431X/14/1/7/pdf?version=1735376342","source":{"id":"https://openalex.org/S4210228075","display_name":"Computers","issn_l":"2073-431X","issn":["2073-431X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computers","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2073-431X/14/1/7/pdf?version=1735376342","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093692725","display_name":"Amirkia Rafiei Oskooei","orcid":"https://orcid.org/0009-0004-3490-550X"},"institutions":[{"id":"https://openalex.org/I4101805","display_name":"Y\u0131ld\u0131z Technical University","ror":"https://ror.org/0547yzj13","country_code":"TR","type":"education","lineage":["https://openalex.org/I4101805"]}],"countries":["TR"],"is_corresponding":true,"raw_author_name":"Amirkia Rafiei Oskooei","raw_affiliation_strings":["Computer Engineering Department, Yildiz Technical University, Istanbul 34320, Turkey"],"raw_orcid":"https://orcid.org/0009-0004-3490-550X","affiliations":[{"raw_affiliation_string":"Computer Engineering Department, Yildiz Technical University, Istanbul 34320, Turkey","institution_ids":["https://openalex.org/I4101805"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036873257","display_name":"Mehmet S. Akta\u015f","orcid":"https://orcid.org/0000-0001-7908-5067"},"institutions":[{"id":"https://openalex.org/I4101805","display_name":"Y\u0131ld\u0131z Technical University","ror":"https://ror.org/0547yzj13","country_code":"TR","type":"education","lineage":["https://openalex.org/I4101805"]}],"countries":["TR"],"is_corresponding":true,"raw_author_name":"Mehmet S. Akta\u015f","raw_affiliation_strings":["Computer Engineering Department, Yildiz Technical University, Istanbul 34320, Turkey"],"raw_orcid":"https://orcid.org/0000-0001-7908-5067","affiliations":[{"raw_affiliation_string":"Computer Engineering Department, Yildiz Technical University, Istanbul 34320, Turkey","institution_ids":["https://openalex.org/I4101805"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5115733540","display_name":"Mustafa Kele\u015f","orcid":null},"institutions":[{"id":"https://openalex.org/I202575966","display_name":"Central Bank of the Republic of Turkey","ror":"https://ror.org/01h4f0m46","country_code":"TR","type":"other","lineage":["https://openalex.org/I202575966"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Mustafa Kele\u015f","raw_affiliation_strings":["Research and Development Center, Aktif Bank, Istanbul 34394, Turkey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Research and Development Center, Aktif Bank, Istanbul 34394, Turkey","institution_ids":["https://openalex.org/I202575966"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5036873257","https://openalex.org/A5093692725"],"corresponding_institution_ids":["https://openalex.org/I4101805"],"apc_list":{"value":1600,"currency":"CHF","value_usd":1732},"apc_paid":{"value":1600,"currency":"CHF","value_usd":1732},"fwci":1.7801,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":{"value":0.86806918,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":"14","issue":"1","first_page":"7","last_page":"7"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12290","display_name":"Human Motion and Animation","score":0.9912999868392944,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/face","display_name":"Face (sociological concept)","score":0.6955693364143372},{"id":"https://openalex.org/keywords/sync","display_name":"sync","score":0.6293769478797913},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.5074579119682312},{"id":"https://openalex.org/keywords/sound","display_name":"Sound (geography)","score":0.5017921924591064},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.46923190355300903},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.46612927317619324},{"id":"https://openalex.org/keywords/communication","display_name":"Communication","score":0.33522653579711914},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.2658948302268982},{"id":"https://openalex.org/keywords/acoustics","display_name":"Acoustics","score":0.21441417932510376},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.1896052062511444},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1717090904712677},{"id":"https://openalex.org/keywords/frame","display_name":"Frame (networking)","score":0.07467985153198242}],"concepts":[{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.6955693364143372},{"id":"https://openalex.org/C3913047","wikidata":"https://www.wikidata.org/wiki/Q1956265","display_name":"sync","level":3,"score":0.6293769478797913},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.5074579119682312},{"id":"https://openalex.org/C203718221","wikidata":"https://www.wikidata.org/wiki/Q491713","display_name":"Sound (geography)","level":2,"score":0.5017921924591064},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.46923190355300903},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.46612927317619324},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.33522653579711914},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2658948302268982},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.21441417932510376},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.1896052062511444},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1717090904712677},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.07467985153198242},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3390/computers14010007","is_oa":true,"landing_page_url":"https://doi.org/10.3390/computers14010007","pdf_url":"https://www.mdpi.com/2073-431X/14/1/7/pdf?version=1735376342","source":{"id":"https://openalex.org/S4210228075","display_name":"Computers","issn_l":"2073-431X","issn":["2073-431X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computers","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:8adbf3447e3846e28c1d86140d9106a1","is_oa":false,"landing_page_url":"https://doaj.org/article/8adbf3447e3846e28c1d86140d9106a1","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computers, Vol 14, Iss 1, p 7 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/computers14010007","is_oa":true,"landing_page_url":"https://doi.org/10.3390/computers14010007","pdf_url":"https://www.mdpi.com/2073-431X/14/1/7/pdf?version=1735376342","source":{"id":"https://openalex.org/S4210228075","display_name":"Computers","issn_l":"2073-431X","issn":["2073-431X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computers","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4405967193.pdf","grobid_xml":"https://content.openalex.org/works/W4405967193.grobid-xml"},"referenced_works_count":75,"referenced_works":["https://openalex.org/W26885468","https://openalex.org/W1502557347","https://openalex.org/W1503933356","https://openalex.org/W1995532034","https://openalex.org/W2018242550","https://openalex.org/W2098561067","https://openalex.org/W2197992444","https://openalex.org/W2331076375","https://openalex.org/W2536201047","https://openalex.org/W2738406145","https://openalex.org/W2804857054","https://openalex.org/W2808631503","https://openalex.org/W2884460600","https://openalex.org/W2898263167","https://openalex.org/W2914217321","https://openalex.org/W2963081548","https://openalex.org/W2981767644","https://openalex.org/W3081492798","https://openalex.org/W3082928416","https://openalex.org/W3101631197","https://openalex.org/W3109114891","https://openalex.org/W3120397982","https://openalex.org/W3130376580","https://openalex.org/W3138644803","https://openalex.org/W3139038810","https://openalex.org/W3156444417","https://openalex.org/W3177150198","https://openalex.org/W3180391059","https://openalex.org/W3187063016","https://openalex.org/W3188692592","https://openalex.org/W3196129874","https://openalex.org/W3204971388","https://openalex.org/W3211147706","https://openalex.org/W4200630629","https://openalex.org/W4212812504","https://openalex.org/W4237389337","https://openalex.org/W4248881089","https://openalex.org/W4297841713","https://openalex.org/W4304087182","https://openalex.org/W4309633564","https://openalex.org/W4312955540","https://openalex.org/W4318148058","https://openalex.org/W4319770604","https://openalex.org/W4321020813","https://openalex.org/W4324057834","https://openalex.org/W4378105483","https://openalex.org/W4382366652","https://openalex.org/W4382469130","https://openalex.org/W4385342469","https://openalex.org/W4385422591","https://openalex.org/W4385626247","https://openalex.org/W4386071653","https://openalex.org/W4386075487","https://openalex.org/W4387843941","https://openalex.org/W4387874311","https://openalex.org/W4389779188","https://openalex.org/W4390154864","https://openalex.org/W4391048004","https://openalex.org/W4392240262","https://openalex.org/W4392903488","https://openalex.org/W4393178509","https://openalex.org/W4393987464","https://openalex.org/W4394597155","https://openalex.org/W4394597549","https://openalex.org/W4400941383","https://openalex.org/W4401548430","https://openalex.org/W4402378552","https://openalex.org/W4402510498","https://openalex.org/W4403064044","https://openalex.org/W4403775178","https://openalex.org/W6604367424","https://openalex.org/W6839079021","https://openalex.org/W6852353527","https://openalex.org/W6852909395","https://openalex.org/W6853040936"],"related_works":["https://openalex.org/W4392306433","https://openalex.org/W2524993630","https://openalex.org/W2063547430","https://openalex.org/W2607244263","https://openalex.org/W2234430299","https://openalex.org/W1831690014","https://openalex.org/W1561022130","https://openalex.org/W1878822803","https://openalex.org/W1995263003","https://openalex.org/W2222580448"],"abstract_inverted_index":{"Imagine":[0],"a":[1,8,33,55,88,107,144,156,170,192,272],"future":[2],"where":[3,278],"language":[4,62,279],"is":[5,125,202,261],"no":[6],"longer":[7],"barrier":[9],"to":[10,127,168,223,228,249,263,281],"real-time":[11,46],"conversations,":[12],"enabling":[13],"instant":[14],"and":[15,68,117,140,154,214,225,258,275],"lifelike":[16],"communication":[17,30,232],"across":[18,83],"the":[19,25,40,75,92,96,269],"globe.":[20],"As":[21],"cultural":[22,259],"boundaries":[23],"blur,":[24],"demand":[26],"for":[27,45,50,137,204,234,271],"seamless":[28],"multilingual":[29,231],"has":[31],"become":[32],"critical":[34],"technological":[35],"challenge.":[36],"This":[37,123,199],"paper":[38],"addresses":[39],"lack":[41],"of":[42,78,98],"robust":[43],"solutions":[44],"face-to-face":[47,173],"translation,":[48],"particularly":[49],"low-resource":[51],"languages,":[52,86,142,243],"by":[53,94,120],"introducing":[54],"comprehensive":[56],"framework":[57,110,124,201],"that":[58,159],"not":[59],"only":[60],"translates":[61],"but":[63],"also":[64],"replicates":[65],"voice":[66],"nuances":[67],"synchronized":[69],"facial":[70],"expressions.":[71],"Our":[72],"research":[73],"tackles":[74],"primary":[76],"challenge":[77],"achieving":[79],"accurate":[80],"lip":[81,99,113,131,162],"synchronization":[82],"culturally":[84],"diverse":[85,235],"filling":[87],"significant":[89],"gap":[90],"in":[91,188,206,255],"literature":[93],"evaluating":[95],"generalizability":[97],"sync":[100,114,132,163],"models":[101,133,164],"beyond":[102],"English.":[103],"Specifically,":[104],"we":[105,152],"develop":[106],"novel":[108],"evaluation":[109],"combining":[111],"quantitative":[112],"error":[115],"metrics":[116],"qualitative":[118],"assessments":[119],"human":[121,283],"observers.":[122],"applied":[126],"assess":[128],"two":[129],"state-of-the-art":[130],"with":[134,165,191],"different":[135],"architectures":[136],"Turkish,":[138],"Persian,":[139],"Arabic":[141],"using":[143],"newly":[145],"collected":[146],"dataset.":[147],"Based":[148],"on":[149,241],"these":[150],"findings,":[151],"propose":[153],"implement":[155],"modular":[157,245],"system":[158,181],"integrates":[160],"language-agnostic":[161],"neural":[166],"networks":[167],"deliver":[169],"fully":[171],"functional":[172],"translation":[174],"experience.":[175],"Inference":[176],"Time":[177],"Analysis":[178],"shows":[179],"this":[180,238],"achieves":[182],"highly":[183],"realistic,":[184],"face-translated":[185],"talking":[186],"heads":[187],"real":[189],"time,":[190],"throughput":[193],"as":[194,196,210],"low":[195],"0.381":[197],"s.":[198],"transformative":[200],"primed":[203],"deployment":[205],"immersive":[207],"environments":[208],"such":[209],"VR/AR,":[211],"Metaverse":[212],"ecosystems,":[213],"advanced":[215],"video":[216],"conferencing":[217],"platforms.":[218],"It":[219],"offers":[220],"substantial":[221],"benefits":[222],"developers":[224],"businesses":[226],"aiming":[227],"build":[229],"next-generation":[230],"systems":[233],"applications.":[236],"While":[237],"work":[239],"focuses":[240],"three":[242],"its":[244,265],"design":[246],"allows":[247],"scalability":[248],"additional":[250],"languages.":[251],"However,":[252],"further":[253],"testing":[254],"broader":[256],"linguistic":[257],"contexts":[260],"required":[262],"confirm":[264],"universal":[266],"applicability,":[267],"paving":[268],"way":[270],"more":[273],"interconnected":[274],"inclusive":[276],"world":[277],"ceases":[280],"hinder":[282],"connection.":[284]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":7}],"updated_date":"2026-05-29T09:21:14.243279","created_date":"2025-10-10T00:00:00"}