{"id":"https://openalex.org/W4415037564","doi":"https://doi.org/10.52202/085713-0381","title":"OmniSync: Towards Universal Lip Synchronization via Diffusion Transformers","display_name":"OmniSync: Towards Universal Lip Synchronization via Diffusion Transformers","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4415037564","doi":"https://doi.org/10.52202/085713-0381"},"language":"en","primary_location":{"id":"doi:10.52202/085713-0381","doi":"https://doi.org/10.52202/085713-0381","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-0381","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2505.21448","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5062816933","display_name":"Ziqiao Peng","orcid":"https://orcid.org/0009-0000-1341-9618","observed_orcids":["https://orcid.org/0009-0000-1341-9618"]},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziqiao Peng","raw_affiliation_strings":["Renmin University of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Renmin University of China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072853454","display_name":"Jiwen Liu","orcid":"https://orcid.org/0000-0002-3798-0714","observed_orcids":["https://orcid.org/0000-0002-3798-0714"]},"institutions":[{"id":"https://openalex.org/I4401726859","display_name":"Kuaishou (China)","ror":"https://ror.org/0258as409","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401726859"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiwen Liu","raw_affiliation_strings":["Kuaishou- \u5feb\u624b\u79d1\u6280"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kuaishou- \u5feb\u624b\u79d1\u6280","institution_ids":["https://openalex.org/I4401726859"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058548349","display_name":"Haoxian Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4401726859","display_name":"Kuaishou (China)","ror":"https://ror.org/0258as409","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401726859"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haoxian Zhang","raw_affiliation_strings":["Kuaishou Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kuaishou Technology","institution_ids":["https://openalex.org/I4401726859"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xiaoqiang Liu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4401726859","display_name":"Kuaishou (China)","ror":"https://ror.org/0258as409","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401726859"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoqiang Liu","raw_affiliation_strings":["Kuaishou- \u5feb\u624b\u79d1\u6280"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kuaishou- \u5feb\u624b\u79d1\u6280","institution_ids":["https://openalex.org/I4401726859"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Songlin Tang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4401726859","display_name":"Kuaishou (China)","ror":"https://ror.org/0258as409","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401726859"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Songlin Tang","raw_affiliation_strings":["Kuaishou"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kuaishou","institution_ids":["https://openalex.org/I4401726859"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101928510","display_name":"Pengfei Wan","orcid":"https://orcid.org/0000-0001-7225-565X","observed_orcids":["https://orcid.org/0000-0001-7225-565X","https://orcid.org/0000-0001-9281-1529","https://orcid.org/0000-0001-5371-1867"]},"institutions":[{"id":"https://openalex.org/I4401726859","display_name":"Kuaishou (China)","ror":"https://ror.org/0258as409","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401726859"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pengfei Wan","raw_affiliation_strings":["Kuaishou Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kuaishou Technology","institution_ids":["https://openalex.org/I4401726859"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Di ZHANG","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4401726859","display_name":"Kuaishou (China)","ror":"https://ror.org/0258as409","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401726859"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Di ZHANG","raw_affiliation_strings":["Kuaishou Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kuaishou Technology","institution_ids":["https://openalex.org/I4401726859"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100332482","display_name":"Hongyan Liu","orcid":"https://orcid.org/0009-0001-8589-2444","observed_orcids":["https://orcid.org/0009-0001-8589-2444"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongyan Liu","raw_affiliation_strings":["Tsinghua University, Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University, Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5049302472","display_name":"Jun He","orcid":"https://orcid.org/0000-0003-1671-2283","observed_orcids":["https://orcid.org/0000-0003-1671-2283"]},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun He","raw_affiliation_strings":["Renmin University of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Renmin University of China","institution_ids":["https://openalex.org/I78988378"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":10.0076,"has_fulltext":true,"cited_by_count":29,"citation_normalized_percentile":{"value":0.98368457,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"12649","last_page":"12668"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9839000105857849,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9839000105857849,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.013500000350177288,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12720","display_name":"Multimedia Communication and Technology","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/2214","display_name":"Media Technology"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/lip-synchronization","display_name":"lip synchronization","score":0.9810000061988831},{"id":"https://openalex.org/keywords/diffusion-transformer","display_name":"diffusion transformer","score":0.6800000071525574},{"id":"https://openalex.org/keywords/flow-matching","display_name":"flow matching","score":0.6809999942779541},{"id":"https://openalex.org/keywords/classifier-free-guidance","display_name":"classifier-free guidance","score":0.8679999709129333},{"id":"https://openalex.org/keywords/identity-consistency","display_name":"identity consistency","score":0.6100000143051147},{"id":"https://openalex.org/keywords/face-occlusion","display_name":"face occlusion","score":0.75},{"id":"https://openalex.org/keywords/ai-generated-video","display_name":"AI-generated video","score":0.6710000038146973},{"id":"https://openalex.org/keywords/audio-visual-alignment","display_name":"audio-visual alignment","score":0.5139999985694885}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7465999722480774},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6237000226974487},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5212000012397766},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.5078999996185303},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4733000099658966},{"id":"https://openalex.org/C3913047","wikidata":"https://www.wikidata.org/wiki/Q1956265","display_name":"sync","level":3,"score":0.4106000065803528},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.4036000072956085},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.4016000032424927},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3637000024318695},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3325999975204468},{"id":"https://openalex.org/C150817343","wikidata":"https://www.wikidata.org/wiki/Q875932","display_name":"Digital watermarking","level":3,"score":0.2888000011444092},{"id":"https://openalex.org/C134652429","wikidata":"https://www.wikidata.org/wiki/Q1052698","display_name":"Jitter","level":2,"score":0.274399995803833},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.27160000801086426},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.26030001044273376},{"id":"https://openalex.org/C187691185","wikidata":"https://www.wikidata.org/wiki/Q2020720","display_name":"Grid","level":2,"score":0.25529998540878296}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52202/085713-0381","doi":"https://doi.org/10.52202/085713-0381","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-0381","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2505.21448","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.21448","pdf_url":"https://arxiv.org/pdf/2505.21448","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.21448","doi":"https://doi.org/10.48550/arxiv.2505.21448","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.21448","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2505.21448","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.21448","pdf_url":"https://arxiv.org/pdf/2505.21448","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415037564.pdf","grobid_xml":"https://content.openalex.org/works/W4415037564.grobid-xml"},"referenced_works_count":27,"referenced_works":["https://openalex.org/W2908510526","https://openalex.org/W3152733922","https://openalex.org/W4288099666","https://openalex.org/W4298185919","https://openalex.org/W4303647933","https://openalex.org/W4388093548","https://openalex.org/W4389115686","https://openalex.org/W4393248189","https://openalex.org/W4399424746","https://openalex.org/W4399695531","https://openalex.org/W4402387301","https://openalex.org/W4403364748","https://openalex.org/W4403555388","https://openalex.org/W4403576235","https://openalex.org/W4405096483","https://openalex.org/W4405355109","https://openalex.org/W4407128586","https://openalex.org/W4407186796","https://openalex.org/W4407713144","https://openalex.org/W4409092337","https://openalex.org/W4413147005","https://openalex.org/W4413155927","https://openalex.org/W4415707885","https://openalex.org/W4417069109","https://openalex.org/W4417499102","https://openalex.org/W6891836174","https://openalex.org/W6891960493"],"related_works":["https://openalex.org/W7117536388","https://openalex.org/W7133949804","https://openalex.org/W4390871677","https://openalex.org/W7134884451","https://openalex.org/W4404446920","https://openalex.org/W7222986536","https://openalex.org/W4407713144","https://openalex.org/W7161701786","https://openalex.org/W4394597155","https://openalex.org/W4402422516"],"abstract_inverted_index":{"Lip":[0],"synchronization":[1,84,178],"is":[2,19],"the":[3,67,142,169,172],"task":[4],"of":[5,146],"aligning":[6],"a":[7,81,93,124,150],"speaker's":[8],"lip":[9,63,72,83,177,197],"movements":[10],"in":[11,179,192,203],"video":[12,25,69],"with":[13],"corresponding":[14],"speech":[15],"audio,":[16,147],"and":[17,35,49,116,132,164,196,206],"it":[18],"essential":[20],"for":[21,86,101,176],"creating":[22],"realistic,":[23],"expressive":[24],"content.":[26,51],"However,":[27],"existing":[28],"methods":[29,191],"often":[30],"rely":[31],"on":[32],"reference":[33],"frames":[34],"masked-frame":[36],"inpainting,":[37],"which":[38],"limit":[39],"their":[40],"robustness":[41],"to":[42,129],"identity":[43,133],"consistency,":[44,134],"pose":[45,131],"variations,":[46],"facial":[47,114],"occlusions,":[48],"stylized":[50],"In":[52,75],"addition,":[53],"since":[54],"audio":[55],"signals":[56],"provide":[57],"weaker":[58],"conditioning":[59,144],"than":[60],"visual":[61,88,194],"cues,":[62],"shape":[64],"leakage":[65],"from":[66],"original":[68],"will":[70],"affect":[71],"sync":[73,198],"quality.":[74],"this":[76],"paper,":[77],"we":[78,122,148],"present":[79],"OmniSync,":[80],"universal":[82],"framework":[85],"diverse":[87,180],"scenarios.":[89],"Our":[90],"approach":[91],"introduces":[92],"mask-free":[94],"training":[95],"paradigm":[96],"using":[97],"Diffusion":[98],"Transformer":[99],"models":[100],"direct":[102],"frame":[103],"editing":[104],"without":[105],"explicit":[106],"masks,":[107],"enabling":[108],"unlimited-duration":[109],"inference":[110],"while":[111,135],"maintaining":[112],"natural":[113],"dynamics":[115],"preserving":[117],"character":[118],"identity.":[119],"During":[120],"inference,":[121],"propose":[123],"flow-matching-based":[125],"progressive":[126],"noise":[127],"initialization":[128],"ensure":[130],"allowing":[136],"precise":[137],"mouth-region":[138],"editing.":[139],"To":[140],"address":[141],"weak":[143],"signal":[145],"develop":[149],"Dynamic":[151],"Spatiotemporal":[152],"Classifier-Free":[153],"Guidance":[154],"(DS-CFG)":[155],"mechanism":[156],"that":[157,186],"adaptively":[158],"adjusts":[159],"guidance":[160],"strength":[161],"over":[162],"time":[163],"space.":[165],"We":[166],"also":[167],"establish":[168],"AIGC-LipSync":[170],"Benchmark,":[171],"first":[173],"evaluation":[174],"suite":[175],"AI-generated":[181,207],"videos.":[182,208],"Extensive":[183],"experiments":[184],"demonstrate":[185],"OmniSync":[187],"significantly":[188],"outperforms":[189],"prior":[190],"both":[193,204],"quality":[195],"accuracy,":[199],"achieving":[200],"superior":[201],"results":[202],"real-world":[205]},"counts_by_year":[{"year":2026,"cited_by_count":18},{"year":2025,"cited_by_count":11}],"updated_date":"2026-11-11T12:16:26.678958","created_date":"2025-10-10T00:00:00"}