{"id":"https://openalex.org/W4404965756","doi":"https://doi.org/10.1145/3680528.3687562","title":"Dance-to-Music Generation with Encoder-based Textual Inversion","display_name":"Dance-to-Music Generation with Encoder-based Textual Inversion","publication_year":2024,"publication_date":"2024-12-03","ids":{"openalex":"https://openalex.org/W4404965756","doi":"https://doi.org/10.1145/3680528.3687562"},"language":"en","primary_location":{"id":"doi:10.1145/3680528.3687562","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3680528.3687562","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3680528.3687562","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SIGGRAPH Asia 2024 Conference Papers","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3680528.3687562","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102871105","display_name":"Sifei Li","orcid":"https://orcid.org/0009-0003-9331-4593"},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sifei Li","raw_affiliation_strings":["MAIS, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0003-9331-4593","affiliations":[{"raw_affiliation_string":"MAIS, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069298091","display_name":"Weiming Dong","orcid":"https://orcid.org/0000-0001-6502-145X"},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weiming Dong","raw_affiliation_strings":["MAIS, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6502-145X","affiliations":[{"raw_affiliation_string":"MAIS, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100398013","display_name":"Yuxin Zhang","orcid":"https://orcid.org/0000-0001-6433-2678"},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuxin Zhang","raw_affiliation_strings":["MAIS, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6433-2678","affiliations":[{"raw_affiliation_string":"MAIS, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112699696","display_name":"Fan Tang","orcid":"https://orcid.org/0000-0002-3975-2483"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fan Tang","raw_affiliation_strings":["University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-3975-2483","affiliations":[{"raw_affiliation_string":"University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025046851","display_name":"Chongyang Ma","orcid":"https://orcid.org/0000-0002-8243-9513"},"institutions":[{"id":"https://openalex.org/I4401726859","display_name":"Kuaishou (China)","ror":"https://ror.org/0258as409","country_code":null,"type":"company","lineage":["https://openalex.org/I4401726859"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chongyang Ma","raw_affiliation_strings":["Kuaishou Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-8243-9513","affiliations":[{"raw_affiliation_string":"Kuaishou Technology, Beijing, China","institution_ids":["https://openalex.org/I4401726859"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036489223","display_name":"Oliver Deu\u00dfen","orcid":"https://orcid.org/0000-0001-5803-2185"},"institutions":[{"id":"https://openalex.org/I189712700","display_name":"University of Konstanz","ror":"https://ror.org/0546hnb39","country_code":"DE","type":"education","lineage":["https://openalex.org/I189712700"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Oliver Deussen","raw_affiliation_strings":["University of Konstanz, Germany, Germany"],"raw_orcid":"https://orcid.org/0000-0001-5803-2185","affiliations":[{"raw_affiliation_string":"University of Konstanz, Germany, Germany","institution_ids":["https://openalex.org/I189712700"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050657606","display_name":"Tong\u2010Yee Lee","orcid":"https://orcid.org/0000-0001-6699-2944"},"institutions":[{"id":"https://openalex.org/I91807558","display_name":"National Cheng Kung University","ror":"https://ror.org/01b8kcc49","country_code":"TW","type":"education","lineage":["https://openalex.org/I91807558"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Tong-Yee Lee","raw_affiliation_strings":["National Cheng-Kung University, Taiwan, Taiwan"],"raw_orcid":"https://orcid.org/0000-0001-6699-2944","affiliations":[{"raw_affiliation_string":"National Cheng-Kung University, Taiwan, Taiwan","institution_ids":["https://openalex.org/I91807558"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5022636178","display_name":"Changsheng Xu","orcid":"https://orcid.org/0000-0001-8343-9665"},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Changsheng Xu","raw_affiliation_strings":["MAIS, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8343-9665","affiliations":[{"raw_affiliation_string":"MAIS, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I4210165038"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12290","display_name":"Human Motion and Animation","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dance","display_name":"Dance","score":0.834498405456543},{"id":"https://openalex.org/keywords/inversion","display_name":"Inversion (geology)","score":0.6664984822273254},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6025152802467346},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5028359293937683},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.3843740224838257},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3525376319885254},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.32693928480148315},{"id":"https://openalex.org/keywords/art","display_name":"Art","score":0.21253162622451782},{"id":"https://openalex.org/keywords/visual-arts","display_name":"Visual arts","score":0.2060489058494568},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.0619560182094574}],"concepts":[{"id":"https://openalex.org/C147446459","wikidata":"https://www.wikidata.org/wiki/Q11639","display_name":"Dance","level":2,"score":0.834498405456543},{"id":"https://openalex.org/C1893757","wikidata":"https://www.wikidata.org/wiki/Q3653001","display_name":"Inversion (geology)","level":3,"score":0.6664984822273254},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6025152802467346},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5028359293937683},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3843740224838257},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3525376319885254},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32693928480148315},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.21253162622451782},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.2060489058494568},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0619560182094574},{"id":"https://openalex.org/C109007969","wikidata":"https://www.wikidata.org/wiki/Q749565","display_name":"Structural basin","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3680528.3687562","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3680528.3687562","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3680528.3687562","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SIGGRAPH Asia 2024 Conference Papers","raw_type":"proceedings-article"},{"id":"pmh:oai:kops.uni-konstanz.de:123456789/71846","is_oa":false,"landing_page_url":"http://nbn-resolving.de/urn:nbn:de:bsz:352-2-9tl19evibu7i0","pdf_url":null,"source":{"id":"https://openalex.org/S4306401487","display_name":"KOPS (University of Konstanz)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I189712700","host_organization_name":"University of Konstanz","host_organization_lineage":["https://openalex.org/I189712700"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IGARASHI, Takeo, Hrsg., Ariel SHAMIR, Hrsg., Hao (Richard) ZHANG, Hrsg.. SIGGRAPH Asia 2024 Conference Papers (SA '24), Proceedings. New York, NY, USA: ACM, 2024, 135. ISBN 979-8-4007-1131-2. Verf\u00fcgbar unter: doi: 10.1145/3680528.3687562","raw_type":"doc-type:conferenceObject"}],"best_oa_location":{"id":"doi:10.1145/3680528.3687562","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3680528.3687562","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3680528.3687562","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SIGGRAPH Asia 2024 Conference Papers","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1017785625","display_name":null,"funder_award_id":"111-2221-E006-112-MY3","funder_id":"https://openalex.org/F2461203286","funder_display_name":"National Science and Technology Council"},{"id":"https://openalex.org/G4735978852","display_name":null,"funder_award_id":"U20B2070, 62102162","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F2461203286","display_name":"National Science and Technology Council","ror":"https://ror.org/02kv4zf79"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4404965756.pdf"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W398859631","https://openalex.org/W569478347","https://openalex.org/W2134163725","https://openalex.org/W2526050071","https://openalex.org/W2811167645","https://openalex.org/W2916798096","https://openalex.org/W3108240585","https://openalex.org/W3204221554","https://openalex.org/W3207290297","https://openalex.org/W4286611264","https://openalex.org/W4312814772","https://openalex.org/W4312933868","https://openalex.org/W4372260310","https://openalex.org/W4385271055","https://openalex.org/W4385287644","https://openalex.org/W4386071613","https://openalex.org/W4386071656","https://openalex.org/W4386071828","https://openalex.org/W4386072096","https://openalex.org/W4386076273","https://openalex.org/W4386076532","https://openalex.org/W4387968791","https://openalex.org/W4387969125","https://openalex.org/W4388994500","https://openalex.org/W4389334989","https://openalex.org/W4389347744","https://openalex.org/W4390873340","https://openalex.org/W4392647990","https://openalex.org/W4393138539","https://openalex.org/W4393148499","https://openalex.org/W4393158405","https://openalex.org/W4396877837","https://openalex.org/W4400965626","https://openalex.org/W4401043564","https://openalex.org/W6607666538"],"related_works":["https://openalex.org/W597595235","https://openalex.org/W3200585538","https://openalex.org/W2357651146","https://openalex.org/W1574267311","https://openalex.org/W2501127304","https://openalex.org/W3209785493","https://openalex.org/W3120492113","https://openalex.org/W2299690913","https://openalex.org/W2762501427","https://openalex.org/W2951440725"],"abstract_inverted_index":{"The":[0],"seamless":[1],"integration":[2],"of":[3,15,27,66,122,131,216],"music":[4,42,74,107],"with":[5,84,102,210],"dance":[6,17,124],"movements":[7],"is":[8,70,199],"essential":[9],"for":[10,75,164],"communicating":[11],"the":[12,24,63,81,85,118,128,169,211,217],"artistic":[13],"intent":[14],"a":[16,123,132,148,176],"piece.":[18],"This":[19],"alignment":[20],"also":[21],"significantly":[22],"improves":[23],"immersive":[25],"quality":[26],"gaming":[28],"experiences":[29],"and":[30,57,120,157,172,184,207,223],"animation":[31],"productions.":[32],"Although":[33],"there":[34],"has":[35],"been":[36],"remarkable":[37],"advancement":[38],"in":[39,72,205],"creating":[40],"high-fidelity":[41],"from":[43],"textual":[44,95,129,138],"descriptions,":[45],"current":[46],"methodologies":[47],"mainly":[48],"focus":[49],"on":[50],"modulating":[51],"overall":[52],"characteristics":[53],"such":[54],"as":[55],"genre":[56,121,158],"emotional":[58],"tone.":[59],"They":[60],"often":[61],"overlook":[62],"nuanced":[64],"management":[65],"temporal":[67],"rhythm,":[68],"which":[69,141],"indispensable":[71],"crafting":[73],"dance,":[76],"since":[77],"it":[78],"intricately":[79],"aligns":[80],"musical":[82],"beats":[83],"dancers\u2019":[86],"movements.":[87],"Recognizing":[88],"this":[89],"gap,":[90],"we":[91,110],"propose":[92],"an":[93],"encoder-based":[94],"inversion":[96,114,139],"technique":[97],"to":[98,115,136,146,160,168,201,203],"augment":[99],"text-to-music":[100,133],"models":[101],"visual":[103],"control,":[104],"facilitating":[105],"personalized":[106],"generation.":[108],"Specifically,":[109],"develop":[111],"dual-path":[112],"rhythm-genre":[113],"effectively":[116,208],"integrate":[117],"rhythm":[119,156],"motion":[125],"sequence":[126],"into":[127],"space":[130],"model.":[134,219],"Contrary":[135],"traditional":[137],"methods,":[140],"directly":[142],"update":[143],"text":[144,162],"embeddings":[145,163],"reconstruct":[147],"single":[149],"target":[150],"object,":[151],"our":[152,187,197],"approach":[153,188],"utilizes":[154],"separate":[155],"encoders":[159],"obtain":[161],"two":[165],"pseudo-words,":[166],"adapting":[167],"varying":[170],"rhythms":[171],"genres.":[173],"We":[174],"collect":[175],"new":[177],"dataset":[178],"called":[179],"In-the-wild":[180],"Dance":[181],"Videos":[182],"(InDV)":[183],"demonstrate":[185],"that":[186],"outperforms":[189],"state-of-the-art":[190],"methods":[191],"across":[192],"multiple":[193],"evaluation":[194],"metrics.":[195],"Furthermore,":[196],"method":[198],"able":[200],"adapt":[202],"changes":[204],"tempo":[206],"integrates":[209],"inherent":[212],"text-guided":[213],"generation":[214],"capability":[215],"pre-trained":[218],"Our":[220],"source":[221],"code":[222],"demo":[224],"videos":[225],"are":[226],"available":[227],"at":[228],"https://github.com/lsfhuihuiff/Dance-to-music_Siggraph_Asia_2024.":[229]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-19T07:52:34.831488","created_date":"2025-10-10T00:00:00"}