{"id":"https://openalex.org/W4400965626","doi":"https://doi.org/10.1007/s41095-024-0417-1","title":"Dance2MIDI: Dance-driven multi-instrument music generation","display_name":"Dance2MIDI: Dance-driven multi-instrument music generation","publication_year":2024,"publication_date":"2024-07-23","ids":{"openalex":"https://openalex.org/W4400965626","doi":"https://doi.org/10.1007/s41095-024-0417-1"},"language":"en","primary_location":{"id":"doi:10.1007/s41095-024-0417-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s41095-024-0417-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s41095-024-0417-1.pdf","source":{"id":"https://openalex.org/S2487656537","display_name":"Computational Visual Media","issn_l":"2096-0433","issn":["2096-0433","2096-0662"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319965","host_organization_name":"Springer Nature","host_organization_lineage":["https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Visual Media","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://link.springer.com/content/pdf/10.1007/s41095-024-0417-1.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101460966","display_name":"Bo Han","orcid":"https://orcid.org/0000-0002-6089-9797"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bo Han","raw_affiliation_strings":["College of Computer Science and Technology, Zhejiang University, Hangzhou 310058, China","College of Computer Science and Technology, Zhejiang University, Hangzhou, 310058, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Hangzhou 310058, China","institution_ids":["https://openalex.org/I76130692"]},{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Hangzhou, 310058, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100652797","display_name":"Yuheng Li","orcid":"https://orcid.org/0009-0001-0249-8952"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuheng Li","raw_affiliation_strings":["College of Computer Science and Technology, Zhejiang University, Hangzhou 310058, China","College of Computer Science and Technology, Zhejiang University, Hangzhou, 310058, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Hangzhou 310058, China","institution_ids":["https://openalex.org/I76130692"]},{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Hangzhou, 310058, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019238907","display_name":"Yixuan Shen","orcid":"https://orcid.org/0000-0002-7560-9326"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yixuan Shen","raw_affiliation_strings":["National University of Singapore, Singapore 119077, Singapore","National University of Singapore, Singapore, 119077, Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore, Singapore 119077, Singapore","institution_ids":["https://openalex.org/I165932596"]},{"raw_affiliation_string":"National University of Singapore, Singapore, 119077, Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088179161","display_name":"Yi Ren","orcid":"https://orcid.org/0000-0001-9889-5460"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yi Ren","raw_affiliation_strings":["Speech & Audio Team, Bytedance AI Lab, Singapore 048583, Singapore","Speech & Audio Team, Bytedance AI Lab, Singapore, 048583, Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Speech & Audio Team, Bytedance AI Lab, Singapore 048583, Singapore","institution_ids":["https://openalex.org/I165932596"]},{"raw_affiliation_string":"Speech & Audio Team, Bytedance AI Lab, Singapore, 048583, Singapore","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008538297","display_name":"Feilin Han","orcid":"https://orcid.org/0000-0001-7463-2252"},"institutions":[{"id":"https://openalex.org/I2802536167","display_name":"Beijing Film Academy","ror":"https://ror.org/03y0w9190","country_code":"CN","type":"education","lineage":["https://openalex.org/I2802536167"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Feilin Han","raw_affiliation_strings":["Department of Film and TV Technology, Beijing Film Academy, Beijing 100088, China","Department of Film and TV Technology, Beijing Film Academy, Beijing, 100088, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Film and TV Technology, Beijing Film Academy, Beijing 100088, China","institution_ids":["https://openalex.org/I2802536167"]},{"raw_affiliation_string":"Department of Film and TV Technology, Beijing Film Academy, Beijing, 100088, China","institution_ids":["https://openalex.org/I2802536167"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5008538297"],"corresponding_institution_ids":["https://openalex.org/I2802536167"],"apc_list":null,"apc_paid":null,"fwci":1.8393,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.86461026,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"10","issue":"4","first_page":"791","last_page":"802"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12290","display_name":"Human Motion and Animation","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dance","display_name":"Dance","score":0.8440999984741211},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4601254463195801},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.43741464614868164},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics","score":0.4129679799079895},{"id":"https://openalex.org/keywords/visual-arts","display_name":"Visual arts","score":0.27863579988479614},{"id":"https://openalex.org/keywords/art","display_name":"Art","score":0.1559913456439972}],"concepts":[{"id":"https://openalex.org/C147446459","wikidata":"https://www.wikidata.org/wiki/Q11639","display_name":"Dance","level":2,"score":0.8440999984741211},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4601254463195801},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.43741464614868164},{"id":"https://openalex.org/C77660652","wikidata":"https://www.wikidata.org/wiki/Q150971","display_name":"Computer graphics","level":2,"score":0.4129679799079895},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.27863579988479614},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.1559913456439972}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s41095-024-0417-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s41095-024-0417-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s41095-024-0417-1.pdf","source":{"id":"https://openalex.org/S2487656537","display_name":"Computational Visual Media","issn_l":"2096-0433","issn":["2096-0433","2096-0662"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319965","host_organization_name":"Springer Nature","host_organization_lineage":["https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Visual Media","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s41095-024-0417-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s41095-024-0417-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s41095-024-0417-1.pdf","source":{"id":"https://openalex.org/S2487656537","display_name":"Computational Visual Media","issn_l":"2096-0433","issn":["2096-0433","2096-0662"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319965","host_organization_name":"Springer Nature","host_organization_lineage":["https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Visual Media","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322725","display_name":"China Scholarship Council","ror":"https://ror.org/04atp4p48"},{"id":"https://openalex.org/F4320335869","display_name":"National Social Science Fund of China","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4400965626.pdf","grobid_xml":"https://content.openalex.org/works/W4400965626.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W1583837637","https://openalex.org/W1847618513","https://openalex.org/W1995875735","https://openalex.org/W2091122332","https://openalex.org/W2163576425","https://openalex.org/W2406198819","https://openalex.org/W2559085405","https://openalex.org/W2772474126","https://openalex.org/W2775487773","https://openalex.org/W2811167645","https://openalex.org/W2916798096","https://openalex.org/W2963076818","https://openalex.org/W3011411500","https://openalex.org/W3043861921","https://openalex.org/W3085146247","https://openalex.org/W3092850823","https://openalex.org/W3094138986","https://openalex.org/W3099425575","https://openalex.org/W3100204393","https://openalex.org/W3108047681","https://openalex.org/W3108240585","https://openalex.org/W3151130473","https://openalex.org/W3204221554","https://openalex.org/W3207290297","https://openalex.org/W4206861281","https://openalex.org/W4233034753","https://openalex.org/W4251070615","https://openalex.org/W4312814772","https://openalex.org/W4313167431","https://openalex.org/W6601202783","https://openalex.org/W6602287985","https://openalex.org/W6602601291","https://openalex.org/W6604428270","https://openalex.org/W6604676790","https://openalex.org/W6629045843"],"related_works":["https://openalex.org/W2319989118","https://openalex.org/W1503820821","https://openalex.org/W2285491073","https://openalex.org/W1991998366","https://openalex.org/W3112751614","https://openalex.org/W1995979513","https://openalex.org/W4249185555","https://openalex.org/W1999937205","https://openalex.org/W2517707608","https://openalex.org/W2144683202"],"abstract_inverted_index":{"Dance-driven":[0],"music":[1,33,56,179,187],"generation":[2,35,84],"aims":[3],"to":[4,92,106,114,118,125,135,172],"generate":[5,126],"musical":[6],"pieces":[7],"conditioned":[8,87],"on":[9,15,76,88,156,193],"dance":[10,71,89,97,109,119,122],"videos.":[11],"Previous":[12],"works":[13],"focus":[14],"monophonic":[16],"or":[17],"raw":[18],"audio":[19],"generation,":[20],"while":[21],"the":[22,52,66,94,108,137,148,152,157,174,177,186,194],"multi-instrument":[23,32,44,68,82],"scenario":[24],"is":[25,170],"under-explored.":[26],"The":[27],"challenges":[28],"associated":[29],"with":[30],"dance-driven":[31],"(MIDI)":[34],"are":[36],"twofold:":[37],"(i)":[38],"lack":[39],"of":[40,150,176],"a":[41,81,102,127,132,142,161],"publicly":[42],"available":[43],"MIDI":[45,69,83],"and":[46,50,57,70,98,121,164,197],"video":[47],"paired":[48,72],"dataset":[49,73,196],"(ii)":[51],"weak":[53],"correlation":[54],"between":[55,96],"video.":[58,90],"To":[59],"tackle":[60],"these":[61],"challenges,":[62],"we":[63,79,100,130,146],"have":[64],"built":[65],"first":[67],"(D2MIDI).":[74],"Based":[75],"this":[77],"dataset,":[78],"introduce":[80],"framework":[85,191],"(Dance2MIDI)":[86],"Firstly,":[91],"capture":[93],"relationship":[95],"music,":[99],"employ":[101],"graph":[103],"convolutional":[104],"network":[105],"encode":[107],"motion.":[110],"This":[111],"allows":[112],"us":[113],"extract":[115],"features":[116],"related":[117],"movement":[120],"style.":[123],"Secondly,":[124],"harmonious":[128],"rhythm,":[129],"utilize":[131],"transformer":[133],"model":[134,147,169],"decode":[136],"drum":[138,158],"track":[139,159],"sequence,":[140],"leveraging":[141],"cross-attention":[143],"mechanism.":[144],"Thirdly,":[145],"task":[149],"generating":[151],"remaining":[153],"tracks":[154],"based":[155],"as":[160],"sequence":[162],"understanding":[163],"completion":[165],"task.":[166],"A":[167],"BERT-like":[168],"employed":[171],"comprehend":[173],"context":[175],"entire":[178],"piece":[180],"through":[181],"self-supervised":[182],"learning.":[183],"We":[184],"evaluate":[185],"generated":[188],"by":[189],"our":[190,200],"trained":[192],"D2MIDI":[195],"demonstrate":[198],"that":[199],"method":[201],"achieves":[202],"state-of-the-art":[203],"performance.":[204]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":3}],"updated_date":"2026-06-13T06:13:01.061226","created_date":"2025-10-10T00:00:00"}