{"id":"https://openalex.org/W4402753940","doi":"https://doi.org/10.1109/cvpr52733.2024.00730","title":"Dysen-VDM: Empowering Dynamics-Aware Text-to-Video Diffusion with LLMs","display_name":"Dysen-VDM: Empowering Dynamics-Aware Text-to-Video Diffusion with LLMs","publication_year":2024,"publication_date":"2024-06-16","ids":{"openalex":"https://openalex.org/W4402753940","doi":"https://doi.org/10.1109/cvpr52733.2024.00730"},"language":"en","primary_location":{"id":"doi:10.1109/cvpr52733.2024.00730","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cvpr52733.2024.00730","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055815455","display_name":"Hao Fei","orcid":null},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Hao Fei","raw_affiliation_strings":["National University of Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089517967","display_name":"Shengqiong Wu","orcid":"https://orcid.org/0000-0001-6192-1194"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Shengqiong Wu","raw_affiliation_strings":["National University of Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101397422","display_name":"Wei Ji","orcid":"https://orcid.org/0000-0002-8106-9768"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Wei Ji","raw_affiliation_strings":["National University of Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042324027","display_name":"Hanwang Zhang","orcid":"https://orcid.org/0000-0001-7374-8739"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hanwang Zhang","raw_affiliation_strings":["Skywork AI,Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Skywork AI,Singapore","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087041599","display_name":"Tat-Seng Chua","orcid":null},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Tat-Seng Chua","raw_affiliation_strings":["National University of Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore","institution_ids":["https://openalex.org/I165932596"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":34,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"7641","last_page":"7653"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12720","display_name":"Multimedia Communication and Technology","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/diffusion","display_name":"Diffusion","score":0.6453964710235596},{"id":"https://openalex.org/keywords/dynamics","display_name":"Dynamics (music)","score":0.6406514644622803},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5971546173095703},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.35937559604644775},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.20031249523162842},{"id":"https://openalex.org/keywords/thermodynamics","display_name":"Thermodynamics","score":0.06313538551330566}],"concepts":[{"id":"https://openalex.org/C69357855","wikidata":"https://www.wikidata.org/wiki/Q163214","display_name":"Diffusion","level":2,"score":0.6453964710235596},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.6406514644622803},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5971546173095703},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.35937559604644775},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.20031249523162842},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.06313538551330566},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/cvpr52733.2024.00730","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cvpr52733.2024.00730","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":106,"referenced_works":["https://openalex.org/W1522734439","https://openalex.org/W2077069816","https://openalex.org/W2425121537","https://openalex.org/W2600702321","https://openalex.org/W2894946340","https://openalex.org/W2902437806","https://openalex.org/W2955233641","https://openalex.org/W2963184176","https://openalex.org/W2963916161","https://openalex.org/W2963966654","https://openalex.org/W2966792645","https://openalex.org/W2976617189","https://openalex.org/W2990138404","https://openalex.org/W2994586138","https://openalex.org/W2994903658","https://openalex.org/W3031246127","https://openalex.org/W3034679267","https://openalex.org/W3035017890","https://openalex.org/W3048484056","https://openalex.org/W3101906322","https://openalex.org/W3115656231","https://openalex.org/W3120341153","https://openalex.org/W3152733922","https://openalex.org/W3153469116","https://openalex.org/W3177821829","https://openalex.org/W3204588463","https://openalex.org/W3207454933","https://openalex.org/W4221143046","https://openalex.org/W4224035735","https://openalex.org/W4224308101","https://openalex.org/W4226278401","https://openalex.org/W4281487633","https://openalex.org/W4281632497","https://openalex.org/W4283009758","https://openalex.org/W4294541506","https://openalex.org/W4297697565","https://openalex.org/W4298185919","https://openalex.org/W4303440777","https://openalex.org/W4309800375","https://openalex.org/W4309801456","https://openalex.org/W4312423208","https://openalex.org/W4312571620","https://openalex.org/W4312633146","https://openalex.org/W4312933868","https://openalex.org/W4317553041","https://openalex.org/W4322718191","https://openalex.org/W4323717348","https://openalex.org/W4327810122","https://openalex.org/W4361029998","https://openalex.org/W4366456807","https://openalex.org/W4377131095","https://openalex.org/W4379912396","https://openalex.org/W4382203549","https://openalex.org/W4384392955","https://openalex.org/W4385245566","https://openalex.org/W4385484740","https://openalex.org/W4385848606","https://openalex.org/W4386066475","https://openalex.org/W4386071957","https://openalex.org/W4386076323","https://openalex.org/W4386320380","https://openalex.org/W4386436185","https://openalex.org/W4387156659","https://openalex.org/W4387635851","https://openalex.org/W4388187930","https://openalex.org/W4388428086","https://openalex.org/W4390872681","https://openalex.org/W4390873135","https://openalex.org/W4390874580","https://openalex.org/W4402667906","https://openalex.org/W6620707391","https://openalex.org/W6718379498","https://openalex.org/W6726983635","https://openalex.org/W6728805014","https://openalex.org/W6745537798","https://openalex.org/W6755353015","https://openalex.org/W6756789066","https://openalex.org/W6763239785","https://openalex.org/W6765779288","https://openalex.org/W6768115207","https://openalex.org/W6771269557","https://openalex.org/W6771861442","https://openalex.org/W6779823529","https://openalex.org/W6781951827","https://openalex.org/W6783713337","https://openalex.org/W6785840567","https://openalex.org/W6787802714","https://openalex.org/W6791353385","https://openalex.org/W6793801364","https://openalex.org/W6798149179","https://openalex.org/W6803369932","https://openalex.org/W6809646742","https://openalex.org/W6810226158","https://openalex.org/W6810793953","https://openalex.org/W6838785959","https://openalex.org/W6838907551","https://openalex.org/W6844305113","https://openalex.org/W6845281891","https://openalex.org/W6846820273","https://openalex.org/W6847095510","https://openalex.org/W6847119409","https://openalex.org/W6848577123","https://openalex.org/W6850071225","https://openalex.org/W6850468791","https://openalex.org/W6852069904","https://openalex.org/W6852746294"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"Text-to-video":[0],"(T2V)":[1],"synthesis":[2],"has":[3],"gained":[4],"increasing":[5],"attention":[6],"in":[7,10,130,200],"the":[8,12,24,55,62,73,106,115,119,128,131,141,158,175],"community,":[9],"which":[11,99],"recently":[13],"emerged":[14],"diffusion":[15],"models":[16],"(DMs)":[17],"have":[18],"promisingly":[19],"shown":[20],"stronger":[21],"performance":[22],"than":[23],"past":[25],"approaches.":[26],"While":[27],"existing":[28,142],"state-of-the-art":[29],"DMs":[30],"are":[31],"competent":[32],"to":[33,54],"achieve":[34],"high-resolution":[35],"video":[36,50,65,76,160,180],"generation,":[37],"they":[38],"may":[39],"largely":[40],"suffer":[41],"from":[42,103],"key":[43,107],"limitations":[44],"(e.g.,":[45,145],"action":[46,116,164],"occurrence":[47],"disorders,":[48],"crude":[49],"motions)":[51],"with":[52,109,133,162,196,202],"respect":[53],"intricate":[56],"temporal":[57,154],"dynamics":[58,77,155],"modeling,":[59],"one":[60],"of":[61,64,75,140],"crux":[63],"synthesis.":[66],"In":[67],"this":[68],"work,":[69],"we":[70,88],"investigate":[71],"strengthening":[72],"awareness":[74],"for":[78,80,179],"DMs,":[79],"high-quality":[81],"T2V":[82,177,185],"generation.":[83],"Inspired":[84],"by":[85],"human":[86],"intuition,":[87],"design":[89],"an":[90],"innovative":[91],"dynamic":[92,120],"scene":[93,121,165],"manager":[94],"(dubbed":[95],"as":[96,169],"Dysen)":[97],"module,":[98],"includes":[100],"(step-1)":[101],"extracting":[102],"input":[104],"text":[105],"actions":[108],"proper":[110],"time-order":[111],"arrangement,":[112],"(step-2)":[113],"transforming":[114],"schedules":[117],"into":[118,174],"graph":[122],"(DSG)":[123],"representations,":[124],"and":[125,135],"(step-3)":[126],"enriching":[127],"scenes":[129],"DSG":[132,161],"sufficient":[134],"reasonable":[136],"details.":[137],"Taking":[138],"advantage":[139],"powerful":[143],"LLMs":[144],"ChatGPT)":[146],"via":[147],"in-context":[148],"learning,":[149],"Dysen":[150],"realizes":[151],"(nearly)":[152],"human-level":[153],"understanding.":[156],"Finally,":[157],"resulting":[159],"rich":[163],"details":[166],"is":[167],"encoded":[168],"fine-grained":[170],"spatio-temporal":[171],"features,":[172],"integrated":[173],"backbone":[176],"DM":[178],"generating.":[181],"Experiments":[182],"on":[183],"popular":[184],"datasets":[186],"suggest":[187],"that":[188],"our":[189],"Dysen-":[190],"VDM":[191],"consistently":[192],"outperforms":[193],"prior":[194],"arts":[195],"significant":[197],"margins,":[198],"especially":[199],"scenarios":[201],"complex":[203],"actions.":[204],"Codes":[205],"at":[206],"http://haofei.vip/Dysen-VDM/.":[207]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":20},{"year":2024,"cited_by_count":13}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}