{"id":"https://openalex.org/W4415540992","doi":"https://doi.org/10.1145/3746027.3755678","title":"Visual-informed Silent Video Identity Conversion","display_name":"Visual-informed Silent Video Identity Conversion","publication_year":2025,"publication_date":"2025-10-25","ids":{"openalex":"https://openalex.org/W4415540992","doi":"https://doi.org/10.1145/3746027.3755678"},"language":"en","primary_location":{"id":"doi:10.1145/3746027.3755678","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3746027.3755678","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2507.00498","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Yifan Liu","orcid":"https://orcid.org/0009-0004-3014-1952"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yifan Liu","raw_affiliation_strings":["LUMIA Lab, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0004-3014-1952","affiliations":[{"raw_affiliation_string":"LUMIA Lab, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yu Fang","orcid":"https://orcid.org/0009-0003-4682-1314"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Fang","raw_affiliation_strings":["LUMIA Lab, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0003-4682-1314","affiliations":[{"raw_affiliation_string":"LUMIA Lab, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024900991","display_name":"Zhouhan Lin","orcid":"https://orcid.org/0009-0009-7204-0689"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhouhan Lin","raw_affiliation_strings":["LUMIA Lab, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0009-7204-0689","affiliations":[{"raw_affiliation_string":"LUMIA Lab, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I183067930"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.29006321,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2104","last_page":"2112"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12357","display_name":"Digital Media Forensic Detection","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12357","display_name":"Digital Media Forensic Detection","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/identity","display_name":"Identity (music)","score":0.6996999979019165},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6284999847412109},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.6022999882698059},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5238999724388123},{"id":"https://openalex.org/keywords/speaker-recognition","display_name":"Speaker recognition","score":0.4950999915599823},{"id":"https://openalex.org/keywords/voice-activity-detection","display_name":"Voice activity detection","score":0.4269999861717224},{"id":"https://openalex.org/keywords/speech-processing","display_name":"Speech processing","score":0.3882000148296356}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7601000070571899},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.7357000112533569},{"id":"https://openalex.org/C2778355321","wikidata":"https://www.wikidata.org/wiki/Q17079427","display_name":"Identity (music)","level":2,"score":0.6996999979019165},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6284999847412109},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.6022999882698059},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5238999724388123},{"id":"https://openalex.org/C133892786","wikidata":"https://www.wikidata.org/wiki/Q1145189","display_name":"Speaker recognition","level":2,"score":0.4950999915599823},{"id":"https://openalex.org/C204201278","wikidata":"https://www.wikidata.org/wiki/Q1332614","display_name":"Voice activity detection","level":3,"score":0.4269999861717224},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.3882000148296356},{"id":"https://openalex.org/C149838564","wikidata":"https://www.wikidata.org/wiki/Q7574248","display_name":"Speaker diarisation","level":3,"score":0.38510000705718994},{"id":"https://openalex.org/C182964821","wikidata":"https://www.wikidata.org/wiki/Q7939498","display_name":"Voice analysis","level":2,"score":0.35670000314712524},{"id":"https://openalex.org/C504749915","wikidata":"https://www.wikidata.org/wiki/Q9010971","display_name":"Speech technology","level":3,"score":0.34439998865127563},{"id":"https://openalex.org/C14999030","wikidata":"https://www.wikidata.org/wiki/Q16346","display_name":"Speech synthesis","level":2,"score":0.34380000829696655},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3133000135421753},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.30329999327659607},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.29840001463890076},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.2632000148296356}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3746027.3755678","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3746027.3755678","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Multimedia","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2507.00498","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2507.00498","pdf_url":"https://arxiv.org/pdf/2507.00498","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2507.00498","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2507.00498","pdf_url":"https://arxiv.org/pdf/2507.00498","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415540992.pdf","grobid_xml":"https://content.openalex.org/works/W4415540992.grobid-xml"},"referenced_works_count":14,"referenced_works":["https://openalex.org/W2808631503","https://openalex.org/W2962788625","https://openalex.org/W3035626590","https://openalex.org/W3081492798","https://openalex.org/W3197659778","https://openalex.org/W3206945595","https://openalex.org/W4372260053","https://openalex.org/W4382202619","https://openalex.org/W4385823403","https://openalex.org/W4386076005","https://openalex.org/W4387969124","https://openalex.org/W4390872449","https://openalex.org/W4402112495","https://openalex.org/W4411113525"],"related_works":[],"abstract_inverted_index":{"Conventional":[0],"voice":[1,4,55],"conversion":[2,56],"modifies":[3],"characteristics":[5],"from":[6,18,58],"a":[7,11,65,69,73,123],"source":[8,74,96],"speaker":[9,67,75,88],"to":[10,130,138,169],"target":[12,66,87],"speaker,":[13],"relying":[14],"on":[15,44,165],"audio":[16,28,166],"input":[17,167],"both":[19,152,174],"sides.":[20],"However,":[21],"this":[22,40,100],"process":[23],"becomes":[24],"infeasible":[25],"when":[26],"clean":[27],"is":[29,114,188],"unavailable,":[30],"such":[31],"as":[32],"in":[33,94,151],"silent":[34,70,97],"videos":[35],"or":[36],"noisy":[37,160],"environments.":[38],"In":[39],"work,":[41],"we":[42,120],"focus":[43],"the":[45,83,86,91,95,175,182],"task":[46,101],"of":[47,64,72,85,177,184],"Silent":[48],"Face-based":[49],"Voice":[50],"Conversion":[51],"(SFVC),":[52],"which":[53],"does":[54],"entirely":[57],"visual":[59,111,141],"inputs.":[60],"i.e.,":[61],"given":[62],"images":[63],"and":[68,106,134,155,181],"video":[71],"containing":[76],"lip":[77],"motion,":[78],"SFVC":[79],"generates":[80],"speech":[81,92,105,153],"aligning":[82],"identity":[84,108,156],"while":[89],"preserving":[90],"content":[93],"video.":[98],"As":[99],"requires":[102],"generating":[103],"intelligible":[104,171],"converting":[107],"using":[109],"only":[110],"cues,":[112],"it":[113],"particularly":[115],"challenging.":[116],"To":[117],"address":[118],"this,":[119],"introduce":[121],"MuteSwap,":[122],"novel":[124],"framework":[125],"that":[126,146],"employs":[127],"contrastive":[128],"learning":[129],"align":[131],"cross-modality":[132],"identities":[133],"minimize":[135],"mutual":[136],"information":[137],"separate":[139],"shared":[140],"features.":[142],"Experimental":[143],"results":[144],"show":[145],"MuteSwap":[147],"achieves":[148],"impressive":[149],"performance":[150],"synthesis":[154],"conversion,":[157],"especially":[158],"under":[159],"conditions":[161],"where":[162],"methods":[163],"dependent":[164],"fail":[168],"produce":[170],"results,":[172],"demonstrating":[173],"effectiveness":[176],"our":[178],"training":[179],"approach":[180],"feasibility":[183],"SFVC.":[185],"Demo":[186],"page":[187],"available":[189],"at":[190],"https://pussycat0700.github.io/MuteSwap-Demo/.":[191]},"counts_by_year":[],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-25T00:00:00"}