{"id":"https://openalex.org/W7155225121","doi":"https://doi.org/10.48550/arxiv.2604.19679","title":"MMControl: Unified Multi-Modal Control for Joint Audio-Video Generation","display_name":"MMControl: Unified Multi-Modal Control for Joint Audio-Video Generation","publication_year":2026,"publication_date":"2026-04-21","ids":{"openalex":"https://openalex.org/W7155225121","doi":"https://doi.org/10.48550/arxiv.2604.19679"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.19679","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.19679","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.19679","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134241259","display_name":"Liyang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Liyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101975688","display_name":"Wen Wang","orcid":"https://orcid.org/0000-0003-4366-933X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Wen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101366798","display_name":"Canyu Zhao","orcid":"https://orcid.org/0009-0002-4142-6844"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Canyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122911381","display_name":"Tianjian Feng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feng, Tianjian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134356412","display_name":"Zhiyue Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Zhiyue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134258083","display_name":"Hao Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134307944","display_name":"Chunhua Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Chunhua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.4512999951839447,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.4512999951839447,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.24709999561309814,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.053199999034404755,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/controllability","display_name":"Controllability","score":0.7257999777793884},{"id":"https://openalex.org/keywords/joint","display_name":"Joint (building)","score":0.6891999840736389},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4756999909877777},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.39559999108314514},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.3456999957561493},{"id":"https://openalex.org/keywords/control-system","display_name":"Control system","score":0.3434999883174896}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7750999927520752},{"id":"https://openalex.org/C48209547","wikidata":"https://www.wikidata.org/wiki/Q1331104","display_name":"Controllability","level":2,"score":0.7257999777793884},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.6891999840736389},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4756999909877777},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.39559999108314514},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3456999957561493},{"id":"https://openalex.org/C17500928","wikidata":"https://www.wikidata.org/wiki/Q959968","display_name":"Control system","level":2,"score":0.3434999883174896},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.33489999175071716},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3215999901294708},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.302700012922287},{"id":"https://openalex.org/C207347870","wikidata":"https://www.wikidata.org/wiki/Q371174","display_name":"Gesture","level":2,"score":0.30169999599456787},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.28450000286102295},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.27219998836517334},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.2711000144481659},{"id":"https://openalex.org/C161759796","wikidata":"https://www.wikidata.org/wiki/Q3982902","display_name":"Teleoperation","level":3,"score":0.2662000060081482},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.2612000107765198}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.19679","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.19679","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.19679","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.19679","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2,57,163],"Diffusion":[3,102],"Transformers":[4],"(DiTs)":[5],"have":[6],"enabled":[7],"high-quality":[8],"joint":[9,58,88,100,164],"audio-video":[10,59,101,165],"generation,":[11],"producing":[12],"videos":[13],"with":[14],"synchronized":[15],"audio":[16,114],"within":[17],"a":[18,63,87,99],"single":[19],"model.":[20],"However,":[21],"existing":[22],"controllable":[23],"generation":[24,89],"frameworks":[25],"are":[26,93],"typically":[27],"restricted":[28],"to":[29,39,53,107,127],"video-only":[30],"control.":[31],"This":[32],"restricts":[33],"comprehensive":[34],"controllability":[35],"and":[36,72,83,112,129,138,160],"often":[37],"leads":[38],"suboptimal":[40],"cross-modal":[41],"alignment.":[42],"To":[43],"bridge":[44],"this":[45],"gap,":[46],"we":[47,119],"present":[48],"MMControl,":[49],"which":[50,124],"enables":[51],"users":[52,126],"perform":[54],"Multi-Modal":[55],"Control":[56],"generation.":[60,166],"MMControl":[61,148],"introduces":[62],"dual-stream":[64],"conditional":[65],"injection":[66],"mechanism.":[67],"It":[68],"incorporates":[69],"both":[70],"visual":[71,137],"acoustic":[73,139],"control":[74,152],"signals,":[75],"including":[76],"reference":[77,79],"images,":[78],"audio,":[80],"depth":[81],"maps,":[82],"pose":[84],"sequences,":[85],"into":[86,98],"process.":[90],"These":[91],"conditions":[92],"injected":[94],"through":[95],"bypass":[96],"branches":[97],"Transformer,":[103],"enabling":[104],"the":[105,132],"model":[106],"simultaneously":[108],"generate":[109],"identity-consistent":[110],"video":[111],"timbre-consistent":[113],"under":[115],"structural":[116],"constraints.":[117],"Furthermore,":[118],"introduce":[120],"modality-specific":[121],"guidance":[122],"scaling,":[123],"allows":[125],"independently":[128],"dynamically":[130],"adjust":[131],"influence":[133],"strength":[134],"of":[135],"each":[136],"condition":[140],"at":[141],"inference":[142],"time.":[143],"Extensive":[144],"experiments":[145],"demonstrate":[146],"that":[147],"achieves":[149],"fine-grained,":[150],"composable":[151],"over":[153],"character":[154],"identity,":[155],"voice":[156],"timbre,":[157],"body":[158],"pose,":[159],"scene":[161],"layout":[162]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-23T00:00:00"}