{"id":"https://openalex.org/W4399554404","doi":"https://doi.org/10.1109/iccv51701.2025.01973","title":"Aid: Adapting Image2video Diffusion Models for Instruction-Guided Video Prediction","display_name":"Aid: Adapting Image2video Diffusion Models for Instruction-Guided Video Prediction","publication_year":2025,"publication_date":"2025-10-19","ids":{"openalex":"https://openalex.org/W4399554404","doi":"https://doi.org/10.1109/iccv51701.2025.01973"},"language":"en","primary_location":{"id":"doi:10.1109/iccv51701.2025.01973","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv51701.2025.01973","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2406.06465","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100297334","display_name":"Zhen Xing","orcid":"https://orcid.org/0009-0008-0050-6323"},"institutions":[{"id":"https://openalex.org/I4210139618","display_name":"Shanghai Key Laboratory of Trustworthy Computing","ror":"https://ror.org/030qbr085","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210139618"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhen Xing","raw_affiliation_strings":["Institute of Trustworthy Embodied AI, Fudan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Trustworthy Embodied AI, Fudan University","institution_ids":["https://openalex.org/I4210139618"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008754580","display_name":"Qi Dai","orcid":"https://orcid.org/0000-0002-9950-6161"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qi Dai","raw_affiliation_strings":["Microsoft Research Asia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Asia","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075142476","display_name":"Zejia Weng","orcid":"https://orcid.org/0000-0001-9706-6484"},"institutions":[{"id":"https://openalex.org/I4210139618","display_name":"Shanghai Key Laboratory of Trustworthy Computing","ror":"https://ror.org/030qbr085","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210139618"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zejia Weng","raw_affiliation_strings":["Institute of Trustworthy Embodied AI, Fudan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Trustworthy Embodied AI, Fudan University","institution_ids":["https://openalex.org/I4210139618"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026167547","display_name":"Zuxuan Wu","orcid":"https://orcid.org/0000-0002-8689-5807"},"institutions":[{"id":"https://openalex.org/I4210139618","display_name":"Shanghai Key Laboratory of Trustworthy Computing","ror":"https://ror.org/030qbr085","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210139618"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zuxuan Wu","raw_affiliation_strings":["Institute of Trustworthy Embodied AI, Fudan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Trustworthy Embodied AI, Fudan University","institution_ids":["https://openalex.org/I4210139618"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047962986","display_name":"Yu\u2013Gang Jiang","orcid":"https://orcid.org/0000-0002-1907-8567"},"institutions":[{"id":"https://openalex.org/I4210139618","display_name":"Shanghai Key Laboratory of Trustworthy Computing","ror":"https://ror.org/030qbr085","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210139618"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu-Gang Jiang","raw_affiliation_strings":["Institute of Trustworthy Embodied AI, Fudan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Trustworthy Embodied AI, Fudan University","institution_ids":["https://openalex.org/I4210139618"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"21243","last_page":"21253"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9915000200271606,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9915000200271606,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9075000286102295,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9021999835968018,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/diffusion","display_name":"Diffusion","score":0.6472461819648743},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6278032660484314},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.32029229402542114},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.06648474931716919}],"concepts":[{"id":"https://openalex.org/C69357855","wikidata":"https://www.wikidata.org/wiki/Q163214","display_name":"Diffusion","level":2,"score":0.6472461819648743},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6278032660484314},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.32029229402542114},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.06648474931716919},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/iccv51701.2025.01973","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv51701.2025.01973","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2406.06465","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.06465","pdf_url":"https://arxiv.org/pdf/2406.06465","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2406.06465","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2406.06465","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2406.06465","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.06465","pdf_url":"https://arxiv.org/pdf/2406.06465","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2478649345","display_name":null,"funder_award_id":"24511103100","funder_id":"https://openalex.org/F4320321885","funder_display_name":"Science and Technology Commission of Shanghai Municipality"},{"id":"https://openalex.org/G5968501908","display_name":null,"funder_award_id":"62032006,62472098","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321885","display_name":"Science and Technology Commission of Shanghai Municipality","ror":"https://ror.org/03kt66j61"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4399554404.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"Text-guided":[0],"video":[1,59,72,86,118,166],"prediction":[2],"(TVP)":[3],"involves":[4],"predicting":[5],"the":[6,12,55,109,139,144],"motion":[7],"of":[8,58],"future":[9,117,148],"frames":[10,123,142],"from":[11],"initial":[13,122],"frame":[14,47,149],"according":[15],"to":[16,54,83,93,115,169],"an":[17],"instruction,":[18],"which":[19,137],"has":[20],"wide":[21],"applications":[22],"in":[23,214],"virtual":[24],"reality,":[25],"robotics,":[26],"and":[27,49,101,124,141,158,196,202,208],"content":[28],"creation.":[29],"Previous":[30],"TVP":[31],"methods":[32],"make":[33],"significant":[34],"breakthroughs":[35],"by":[36],"adapting":[37],"Stable":[38],"Diffusion":[39],"for":[40,71,147],"this":[41],"task.":[42,103],"However,":[43],"they":[44,75],"struggle":[45],"with":[46,172],"consistency":[48],"temporal":[50],"stability":[51],"primarily":[52],"due":[53],"limited":[56],"scale":[57],"datasets.":[60],"We":[61],"observe":[62],"that":[63,161,179],"pretrained":[64],"Image2Video":[65,81],"diffusion":[66,167],"models":[67,82,168],"possess":[68],"good":[69],"priors":[70,88],"dynamics":[73],"but":[74],"lack":[76],"textual":[77],"control.":[78],"Hence,":[79],"transferring":[80],"leverage":[84],"their":[85],"dynamic":[87],"while":[89],"injecting":[90],"instruction":[91],"control":[92],"generate":[94],"controllable":[95],"videos":[96],"is":[97],"both":[98],"a":[99,131],"meaningful":[100],"challenging":[102],"To":[104],"achieve":[105],"this,":[106],"we":[107,129,152],"introduce":[108],"Multi-Modal":[110],"Large":[111],"Language":[112],"Model":[113],"(MLLM)":[114],"predict":[116],"states":[119],"based":[120],"on":[121,186,206],"text":[125],"instructions.":[126],"More":[127,217],"specifically,":[128],"design":[130],"dual":[132],"query":[133],"transformer":[134],"(DQFormer)":[135],"architecture,":[136],"integrates":[138],"instructions":[140],"into":[143],"conditional":[145],"embeddings":[146],"prediction.":[150],"Additionally,":[151],"develop":[153],"Long-Short":[154],"Term":[155],"Temporal":[156],"Adapters":[157,160],"Spatial":[159],"can":[162,219],"quickly":[163],"transfer":[164],"general":[165],"specific":[170],"scenarios":[171],"minimal":[173],"training":[174],"costs.":[175],"Experimental":[176],"results":[177],"show":[178],"our":[180,223],"method":[181],"significantly":[182],"outperforms":[183],"state-of-the-art":[184],"techniques":[185],"four":[187],"datasets:":[188],"Something":[189,190],"V2,":[191],"Epic":[192],"Kitchen-100,":[193],"Bridge":[194,207],"Data,":[195],"UCF-101.":[197],"Notably,":[198],"AID":[199],"achieves":[200],"91.2%":[201],"55.5%":[203],"FVD":[204],"improvements":[205],"SSv2":[209],"respectively,":[210],"demonstrating":[211],"its":[212],"effectiveness":[213],"various":[215],"domains.":[216],"examples":[218],"be":[220],"found":[221],"at":[222],"website":[224],"https://chenhsing.github.io/AID.":[225]},"counts_by_year":[{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}