{"id":"https://openalex.org/W4402980762","doi":"https://doi.org/10.1109/icme57554.2024.10687925","title":"SingAvatar: High-fidelity Audio-driven Singing Avatar Synthesis","display_name":"SingAvatar: High-fidelity Audio-driven Singing Avatar Synthesis","publication_year":2024,"publication_date":"2024-07-15","ids":{"openalex":"https://openalex.org/W4402980762","doi":"https://doi.org/10.1109/icme57554.2024.10687925"},"language":"en","primary_location":{"id":"doi:10.1109/icme57554.2024.10687925","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/icme57554.2024.10687925","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Multimedia and Expo (ICME)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003738065","display_name":"Wentao Ma","orcid":"https://orcid.org/0000-0002-0299-3486"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wentao Ma","raw_affiliation_strings":["Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006595285","display_name":"Anni Tang","orcid":"https://orcid.org/0000-0002-9772-3293"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Anni Tang","raw_affiliation_strings":["Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103020698","display_name":"Jun Ling","orcid":"https://orcid.org/0000-0001-7260-7141"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Ling","raw_affiliation_strings":["Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100748307","display_name":"Han Xue","orcid":"https://orcid.org/0000-0003-4699-4701"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Han Xue","raw_affiliation_strings":["Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112891042","display_name":"Huiheng Liao","orcid":null},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huiheng Liao","raw_affiliation_strings":["Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017705206","display_name":"Yunhui Zhu","orcid":"https://orcid.org/0000-0003-4323-9102"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yunhui Zhu","raw_affiliation_strings":["Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100448224","display_name":"Song Li","orcid":"https://orcid.org/0000-0003-2163-8452"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li Song","raw_affiliation_strings":["Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University,School of Electronic Information and Electrical Engineering,Shanghai,China","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I183067930"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.9822999835014343,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.9822999835014343,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.948199987411499,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9300000071525574,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/avatar","display_name":"Avatar","score":0.8343155384063721},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7700977325439453},{"id":"https://openalex.org/keywords/high-fidelity","display_name":"High fidelity","score":0.7330827116966248},{"id":"https://openalex.org/keywords/singing","display_name":"Singing","score":0.6270452737808228},{"id":"https://openalex.org/keywords/fidelity","display_name":"Fidelity","score":0.591663122177124},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4462394714355469},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.39498215913772583},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.38505688309669495},{"id":"https://openalex.org/keywords/acoustics","display_name":"Acoustics","score":0.12271907925605774},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.09627878665924072}],"concepts":[{"id":"https://openalex.org/C2777365542","wikidata":"https://www.wikidata.org/wiki/Q83090","display_name":"Avatar","level":2,"score":0.8343155384063721},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7700977325439453},{"id":"https://openalex.org/C113364801","wikidata":"https://www.wikidata.org/wiki/Q26674","display_name":"High fidelity","level":2,"score":0.7330827116966248},{"id":"https://openalex.org/C44819458","wikidata":"https://www.wikidata.org/wiki/Q27939","display_name":"Singing","level":2,"score":0.6270452737808228},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.591663122177124},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4462394714355469},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.39498215913772583},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.38505688309669495},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.12271907925605774},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.09627878665924072},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icme57554.2024.10687925","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/icme57554.2024.10687925","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Multimedia and Expo (ICME)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320327718","display_name":"Shanghai Key Laboratory of Digital Media Processing and Transmission","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W2133665775","https://openalex.org/W2237250383","https://openalex.org/W2604379605","https://openalex.org/W2747874407","https://openalex.org/W2769666294","https://openalex.org/W2804619907","https://openalex.org/W2806833697","https://openalex.org/W2933138175","https://openalex.org/W2962785568","https://openalex.org/W2963081548","https://openalex.org/W3081492798","https://openalex.org/W3097792222","https://openalex.org/W3106759947","https://openalex.org/W3109585842","https://openalex.org/W3158762648","https://openalex.org/W3180794345","https://openalex.org/W3204715535","https://openalex.org/W4385245566","https://openalex.org/W4386072021"],"related_works":["https://openalex.org/W4313443006","https://openalex.org/W2945374968","https://openalex.org/W4385452045","https://openalex.org/W4293777179","https://openalex.org/W2164070813","https://openalex.org/W2135608140","https://openalex.org/W2895525995","https://openalex.org/W4224231624","https://openalex.org/W2332512904","https://openalex.org/W2319626700"],"abstract_inverted_index":{"Generating":[0],"photo-realistic":[1],"avatars":[2],"from":[3,23,97],"audio":[4,22,100,118],"plays":[5],"an":[6],"important":[7],"role":[8],"in":[9],"extended":[10],"reality":[11],"(XR)":[12],"and":[13,37,74,94,101,119,137,144],"metaverse.":[14],"In":[15,123],"this":[16],"paper,":[17],"we":[18,55,68,86,106],"lift":[19],"the":[20,49,66,78,82,98,132,140],"input":[21],"speech":[24],"to":[25,48,76],"singing,":[26],"which":[27],"has":[28],"been":[29],"rarely":[30],"studied.":[31],"The":[32,142],"significant":[33],"distinction":[34],"between":[35],"singing":[36,50,59,91,110,114],"talking":[38,44],"poses":[39],"great":[40],"challenges":[41],"for":[42],"adapting":[43],"face":[45],"generation":[46,96],"methods":[47,128],"regime.":[51],"To":[52],"address":[53],"this,":[54],"propose":[56],"a":[57,88,108],"high-fidelity":[58],"avatar":[60],"synthesis":[61,93],"method":[62],"called":[63],"SingAvatar.":[64],"Besides":[65],"audio,":[67],"incorporate":[69],"vocal":[70,103,121],"conditions":[71],"involving":[72],"phonemes":[73],"variance":[75],"alleviate":[77],"ambiguity":[79],"of":[80,134],"learning":[81],"singing-to-face":[83],"mapping.":[84],"Concretely,":[85],"tailor":[87],"two-stage":[89],"pipeline:":[90],"voice":[92],"portrait":[95],"synthesized":[99],"auxiliary":[102],"conditions.":[104,122],"Further,":[105],"curate":[107],"fine-grained":[109],"head":[111,135],"dataset":[112,145],"containing":[113],"videos":[115],"with":[116],"synchronized":[117],"accurate":[120],"experiments,":[124],"SingAvatar":[125],"outperforms":[126],"competing":[127],"regarding":[129],"audio-mouth":[130],"synchronization,":[131],"naturalness":[133],"movements,":[136],"controllability":[138],"over":[139],"results.":[141],"code":[143],"will":[146],"be":[147],"made":[148],"publicly":[149],"available.":[150]},"counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}