{"id":"https://openalex.org/W7147025202","doi":"https://doi.org/10.48550/arxiv.2603.29162","title":"From Natural Alignment to Conditional Controllability in Multimodal Dialogue","display_name":"From Natural Alignment to Conditional Controllability in Multimodal Dialogue","publication_year":2026,"publication_date":"2026-03-31","ids":{"openalex":"https://openalex.org/W7147025202","doi":"https://doi.org/10.48550/arxiv.2603.29162"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.29162","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.29162","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.29162","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5132589898","display_name":"Zeyu Jin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Zeyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026277472","display_name":"Songtao Zhou","orcid":"https://orcid.org/0009-0008-5972-3955"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Songtao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132597889","display_name":"Haoyu Albert Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Haoyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132545615","display_name":"Minghao Tian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tian, Minghao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113392275","display_name":"Kaifeng Yun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yun, Kaifeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132581962","display_name":"Zhuo Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Zhuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101425035","display_name":"Xiaoyu Qin","orcid":"https://orcid.org/0000-0003-3877-6376"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qin, Xiaoyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5132541342","display_name":"Jia Jia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jia, Jia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9146999716758728,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9146999716758728,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.025699999183416367,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.01810000091791153,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5942999720573425},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.5038999915122986},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.5004000067710876},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.49309998750686646},{"id":"https://openalex.org/keywords/conditional-random-field","display_name":"Conditional random field","score":0.46540001034736633},{"id":"https://openalex.org/keywords/multimodal-interaction","display_name":"Multimodal interaction","score":0.45419999957084656},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.44020000100135803},{"id":"https://openalex.org/keywords/multimodality","display_name":"Multimodality","score":0.4016999900341034},{"id":"https://openalex.org/keywords/controllability","display_name":"Controllability","score":0.40139999985694885}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7584999799728394},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5942999720573425},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5307999849319458},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.5038999915122986},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5004000067710876},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.49309998750686646},{"id":"https://openalex.org/C152565575","wikidata":"https://www.wikidata.org/wiki/Q1124538","display_name":"Conditional random field","level":2,"score":0.46540001034736633},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.45419999957084656},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.44020000100135803},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4277999997138977},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.4016999900341034},{"id":"https://openalex.org/C48209547","wikidata":"https://www.wikidata.org/wiki/Q1331104","display_name":"Controllability","level":2,"score":0.40139999985694885},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.37929999828338623},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.374099999666214},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.36910000443458557},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.3646000027656555},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.34119999408721924},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.3345000147819519},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.32280001044273376},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.3089999854564667},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.29910001158714294},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.29120001196861267},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.27959999442100525},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.27630001306533813},{"id":"https://openalex.org/C207347870","wikidata":"https://www.wikidata.org/wiki/Q371174","display_name":"Gesture","level":2,"score":0.27619999647140503},{"id":"https://openalex.org/C2775852435","wikidata":"https://www.wikidata.org/wiki/Q258403","display_name":"Utterance","level":2,"score":0.27489998936653137},{"id":"https://openalex.org/C145460709","wikidata":"https://www.wikidata.org/wiki/Q859951","display_name":"Human\u2013robot interaction","level":3,"score":0.2720000147819519},{"id":"https://openalex.org/C30539005","wikidata":"https://www.wikidata.org/wiki/Q1066689","display_name":"Human communication","level":2,"score":0.2621999979019165},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.2540999948978424}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.29162","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.29162","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.29162","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.29162","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0,106],"recent":[1],"advancement":[2],"of":[3,22,78,176],"Artificial":[4],"Intelligence":[5],"Generated":[6],"Content":[7],"(AIGC)":[8],"has":[9],"led":[10],"to":[11,92,171],"significant":[12],"strides":[13],"in":[14,19,32,41,59,81,103,168],"modeling":[15],"human":[16,60,177],"interaction,":[17,61],"particularly":[18],"the":[20,51,73,173],"context":[21],"multimodal":[23,68,88,187],"dialogue.":[24],"While":[25],"current":[26,169],"methods":[27],"impressively":[28],"generate":[29],"realistic":[30],"dialogue":[31,65,79,89,121,189],"isolated":[33],"modalities":[34],"like":[35],"speech":[36,122],"or":[37],"vision,":[38,56],"challenges":[39,185],"remain":[40],"controllable":[42],"Multimodal":[43],"Dialogue":[44],"Generation":[45],"(MDG).":[46],"This":[47],"paper":[48],"focuses":[49],"on":[50,156,164],"natural":[52],"alignment":[53],"between":[54],"speech,":[55],"and":[57,76,97,184],"text":[58],"aiming":[62],"for":[63,140,186],"expressive":[64,129],"generation":[66],"through":[67,119],"conditional":[69,188],"control.":[70],"To":[71],"address":[72],"insufficient":[74],"richness":[75],"diversity":[77],"expressiveness":[80,175],"existing":[82],"datasets,":[83],"we":[84],"introduce":[85],"a":[86,137],"novel":[87],"annotation":[90],"pipeline":[91],"curate":[93],"dialogues":[94,130],"from":[95],"movies":[96],"TV":[98],"series":[99],"with":[100,131],"fine-grained":[101,160],"annotations":[102],"interactional":[104],"characteristics.":[105],"resulting":[107],"MM-Dia":[108,157],"dataset":[109],"(360+":[110],"hours,":[111],"54,700":[112],"dialogues)":[113],"facilitates":[114],"explicitly":[115],"controlled":[116],"MDG,":[117],"specifically":[118],"style-controllable":[120],"synthesis.":[123],"In":[124],"parallel,":[125],"MM-Dia-Bench":[126,165],"(309":[127],"highly":[128],"visible":[132],"single-/dual-speaker":[133],"scenes)":[134],"serves":[135],"as":[136],"rigorous":[138],"testbed":[139],"implicit":[141],"cross-modal":[142],"MDG":[143],"control,":[144],"evaluating":[145],"audio-visual":[146],"style":[147],"consistency":[148],"across":[149],"modalities.":[150],"Extensive":[151],"experiments":[152],"demonstrate":[153],"that":[154],"training":[155],"significantly":[158],"enhances":[159],"controllability,":[161],"while":[162],"evaluations":[163],"reveal":[166],"limitations":[167],"frameworks":[170],"replicate":[172],"nuanced":[174],"interaction.":[178],"These":[179],"findings":[180],"provides":[181],"new":[182],"insights":[183],"generation.":[190]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-04-02T00:00:00"}