{"id":"https://openalex.org/W4392903560","doi":"https://doi.org/10.1109/icassp48485.2024.10447284","title":"Exploring Phonetic Context-Aware Lip-Sync for Talking Face Generation","display_name":"Exploring Phonetic Context-Aware Lip-Sync for Talking Face Generation","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4392903560","doi":"https://doi.org/10.1109/icassp48485.2024.10447284"},"language":"en","primary_location":{"id":"doi:10.1109/icassp48485.2024.10447284","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10447284","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100376689","display_name":"Se Jin Park","orcid":"https://orcid.org/0000-0002-1618-5906"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]},{"id":"https://openalex.org/I4210107233","display_name":"Language Science (South Korea)","ror":"https://ror.org/01h9v1373","country_code":"KR","type":"company","lineage":["https://openalex.org/I4210107233"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Se Jin Park","raw_affiliation_strings":["KAIST,Integrated Vision and Language Lab. School of EE,South Korea","Integrated Vision and Language Lab. School of EE, KAIST, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAIST,Integrated Vision and Language Lab. School of EE,South Korea","institution_ids":["https://openalex.org/I4210107233"]},{"raw_affiliation_string":"Integrated Vision and Language Lab. School of EE, KAIST, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100343629","display_name":"Minsu Kim","orcid":"https://orcid.org/0000-0002-6514-0018"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]},{"id":"https://openalex.org/I4210107233","display_name":"Language Science (South Korea)","ror":"https://ror.org/01h9v1373","country_code":"KR","type":"company","lineage":["https://openalex.org/I4210107233"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Minsu Kim","raw_affiliation_strings":["KAIST,Integrated Vision and Language Lab. School of EE,South Korea","Integrated Vision and Language Lab. School of EE, KAIST, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAIST,Integrated Vision and Language Lab. School of EE,South Korea","institution_ids":["https://openalex.org/I4210107233"]},{"raw_affiliation_string":"Integrated Vision and Language Lab. School of EE, KAIST, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101725649","display_name":"Jeongsoo Choi","orcid":"https://orcid.org/0009-0005-6817-604X"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]},{"id":"https://openalex.org/I4210107233","display_name":"Language Science (South Korea)","ror":"https://ror.org/01h9v1373","country_code":"KR","type":"company","lineage":["https://openalex.org/I4210107233"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jeongsoo Choi","raw_affiliation_strings":["KAIST,Integrated Vision and Language Lab. School of EE,South Korea","Integrated Vision and Language Lab. School of EE, KAIST, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAIST,Integrated Vision and Language Lab. School of EE,South Korea","institution_ids":["https://openalex.org/I4210107233"]},{"raw_affiliation_string":"Integrated Vision and Language Lab. School of EE, KAIST, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5038798134","display_name":"Yong Man Ro","orcid":"https://orcid.org/0000-0001-5306-6853"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]},{"id":"https://openalex.org/I4210107233","display_name":"Language Science (South Korea)","ror":"https://ror.org/01h9v1373","country_code":"KR","type":"company","lineage":["https://openalex.org/I4210107233"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Yong Man Ro","raw_affiliation_strings":["KAIST,Integrated Vision and Language Lab. School of EE,South Korea","Integrated Vision and Language Lab. School of EE, KAIST, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAIST,Integrated Vision and Language Lab. School of EE,South Korea","institution_ids":["https://openalex.org/I4210107233"]},{"raw_affiliation_string":"Integrated Vision and Language Lab. School of EE, KAIST, South Korea","institution_ids":["https://openalex.org/I157485424"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4325","last_page":"4329"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7605481743812561},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.7317836284637451},{"id":"https://openalex.org/keywords/articulation","display_name":"Articulation (sociology)","score":0.7190654873847961},{"id":"https://openalex.org/keywords/phone","display_name":"Phone","score":0.6622499227523804},{"id":"https://openalex.org/keywords/face","display_name":"Face (sociological concept)","score":0.6374983787536621},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6212974786758423},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.5754976272583008},{"id":"https://openalex.org/keywords/sync","display_name":"sync","score":0.5704091787338257},{"id":"https://openalex.org/keywords/motion","display_name":"Motion (physics)","score":0.5019514560699463},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.4318792223930359},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.415862500667572},{"id":"https://openalex.org/keywords/frame","display_name":"Frame (networking)","score":0.2552606463432312},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.10556569695472717},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.10461625456809998}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7605481743812561},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.7317836284637451},{"id":"https://openalex.org/C2779337067","wikidata":"https://www.wikidata.org/wiki/Q4800961","display_name":"Articulation (sociology)","level":3,"score":0.7190654873847961},{"id":"https://openalex.org/C2778707766","wikidata":"https://www.wikidata.org/wiki/Q202064","display_name":"Phone","level":2,"score":0.6622499227523804},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.6374983787536621},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6212974786758423},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.5754976272583008},{"id":"https://openalex.org/C3913047","wikidata":"https://www.wikidata.org/wiki/Q1956265","display_name":"sync","level":3,"score":0.5704091787338257},{"id":"https://openalex.org/C104114177","wikidata":"https://www.wikidata.org/wiki/Q79782","display_name":"Motion (physics)","level":2,"score":0.5019514560699463},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4318792223930359},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.415862500667572},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.2552606463432312},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.10556569695472717},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.10461625456809998},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp48485.2024.10447284","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10447284","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2177770941","display_name":null,"funder_award_id":"4120200113769","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"},{"id":"https://openalex.org/G30685149","display_name":null,"funder_award_id":"BK21 FOUR","funder_id":"https://openalex.org/F4320320671","funder_display_name":"National Research Foundation"},{"id":"https://openalex.org/G5178579001","display_name":null,"funder_award_id":"NRF-2022R1A2C2005529","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"},{"id":"https://openalex.org/G5562307789","display_name":null,"funder_award_id":"BK21 FOUR","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"},{"id":"https://openalex.org/G6962113442","display_name":null,"funder_award_id":"4120200113769","funder_id":"https://openalex.org/F4320324161","funder_display_name":"Korea Advanced Institute of Science and Technology"}],"funders":[{"id":"https://openalex.org/F4320320671","display_name":"National Research Foundation","ror":"https://ror.org/05s0g1g46"},{"id":"https://openalex.org/F4320322120","display_name":"National Research Foundation of Korea","ror":"https://ror.org/013aysd81"},{"id":"https://openalex.org/F4320324161","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63"},{"id":"https://openalex.org/F4320328359","display_name":"Ministry of Science and ICT, South Korea","ror":"https://ror.org/01wpjm123"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":43,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1920769845","https://openalex.org/W2149799518","https://openalex.org/W2170186674","https://openalex.org/W2594690981","https://openalex.org/W2604379605","https://openalex.org/W2808631503","https://openalex.org/W2890952074","https://openalex.org/W2896457183","https://openalex.org/W2914217321","https://openalex.org/W2962766044","https://openalex.org/W2963081548","https://openalex.org/W2963290645","https://openalex.org/W2964559396","https://openalex.org/W2979894294","https://openalex.org/W2981767644","https://openalex.org/W2988975212","https://openalex.org/W3015734344","https://openalex.org/W3016663370","https://openalex.org/W3023351797","https://openalex.org/W3024308166","https://openalex.org/W3034775979","https://openalex.org/W3081492798","https://openalex.org/W3097777922","https://openalex.org/W3101631197","https://openalex.org/W3105763085","https://openalex.org/W3186090335","https://openalex.org/W3187364420","https://openalex.org/W3195529437","https://openalex.org/W3197199219","https://openalex.org/W3211862173","https://openalex.org/W4226204502","https://openalex.org/W4283818626","https://openalex.org/W4297841411","https://openalex.org/W4297841713","https://openalex.org/W4308663722","https://openalex.org/W4385245566","https://openalex.org/W6631190155","https://openalex.org/W6755207826","https://openalex.org/W6773829392","https://openalex.org/W6803359641","https://openalex.org/W6810980340","https://openalex.org/W6811548253"],"related_works":["https://openalex.org/W2063547430","https://openalex.org/W2524993630","https://openalex.org/W2607244263","https://openalex.org/W2234430299","https://openalex.org/W1831690014","https://openalex.org/W1561022130","https://openalex.org/W1878822803","https://openalex.org/W1995263003","https://openalex.org/W2222580448","https://openalex.org/W4206700937"],"abstract_inverted_index":{"Talking":[0],"face":[1,13,76],"generation":[2,186],"is":[3,29,98,110],"the":[4,32,37,44,52,67,94,133,152,156,167,171,180],"challenging":[5],"task":[6],"of":[7,39,93,100],"synthesizing":[8,136],"a":[9,19,40,105,121,137],"natural":[10],"and":[11,104,178],"realistic":[12],"that":[14,149],"requires":[15],"accurate":[16],"synchronization":[17,177],"with":[18,51,140],"given":[20],"audio.":[21],"Due":[22],"to":[23,89,116,120,169,187],"co-articulation,":[24],"where":[25],"an":[26,101],"isolated":[27],"phone":[28,41,119],"influenced":[30],"by":[31],"preceding":[33],"or":[34],"following":[35],"phones,":[36],"articulation":[38],"varies":[42],"upon":[43],"phonetic":[45,53,68,87,153,172],"context.":[46],"Therefore,":[47],"modeling":[48],"lip":[49,60,72,91,123,128,142,176,185],"motion":[50,73,124,129],"context":[54,69,88,154,173],"can":[55],"generate":[56,90],"more":[57],"spatio-temporally":[58],"aligned":[59],"movement.":[61],"In":[62],"this":[63],"respect,":[64],"we":[65,147],"investigate":[66],"in":[70,135,155,175],"generating":[71],"for":[74,184],"talking":[75],"generation.":[77],"We":[78,164],"propose":[79],"Context-Aware":[80],"Lip-Sync":[81],"framework":[82,159],"(CALS),":[83],"which":[84,170],"explicitly":[85],"leverages":[86],"movement":[92],"target":[95,138],"face.":[96],"CALS":[97,158],"comprised":[99],"Audio-to-Lip":[102],"module":[103],"Lip-to-Face":[106],"module.":[107],"The":[108,126],"former":[109],"pretrained":[111],"based":[112],"on":[113],"masked":[114],"learning":[115],"map":[117],"each":[118],"contextualized":[122,127],"unit.":[125],"unit":[130],"then":[131],"guides":[132],"latter":[134],"identity":[139],"context-aware":[141],"motion.":[143],"From":[144],"extensive":[145],"experiments,":[146],"verify":[148],"simply":[150],"exploiting":[151],"proposed":[157],"effectively":[160],"enhances":[161],"spatio-temporal":[162],"alignment.":[163],"also":[165],"demonstrate":[166],"extent":[168],"assists":[174],"find":[179],"effective":[181],"window":[182],"size":[183],"be":[188],"approximately":[189],"1.2":[190],"seconds.":[191]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}