{"id":"https://openalex.org/W7130690807","doi":"https://doi.org/10.36227/techrxiv.177162137.77131035/v1","title":"Empowering Video Translation using Multimodal Large Language Models","display_name":"Empowering Video Translation using Multimodal Large Language Models","publication_year":2026,"publication_date":"2026-02-20","ids":{"openalex":"https://openalex.org/W7130690807","doi":"https://doi.org/10.36227/techrxiv.177162137.77131035/v1"},"language":"en","primary_location":{"id":"doi:10.36227/techrxiv.177162137.77131035/v1","doi":"https://doi.org/10.36227/techrxiv.177162137.77131035/v1","is_oa":true,"landing_page_url":"https://doi.org/10.36227/techrxiv.177162137.77131035/v1","pdf_url":"https://www.techrxiv.org/doi/pdf/10.36227/techrxiv.177162137.77131035","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"volume":null,"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.techrxiv.org/doi/pdf/10.36227/techrxiv.177162137.77131035","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5123961460","display_name":"Bingzheng Qu","orcid":"https://orcid.org/0009-0002-2923-7135","observed_orcids":["https://orcid.org/0009-0002-2923-7135"]},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Bingzheng Qu","raw_affiliation_strings":["School of Computer Science and Technol- ogy , Harbin Institute of Technology , Shenzhen 518055 , China"],"raw_orcid":"https://orcid.org/0009-0002-2923-7135","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technol- ogy , Harbin Institute of Technology , Shenzhen 518055 , China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006323375","display_name":"Kehai Chen","orcid":"https://orcid.org/0000-0002-4346-7618","observed_orcids":["https://orcid.org/0000-0002-4346-7618"]},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kehai Chen","raw_affiliation_strings":["School of Computer Science and Technol- ogy , Harbin Institute of Technology , Shenzhen 518055 , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Technol- ogy , Harbin Institute of Technology , Shenzhen 518055 , China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123898467","display_name":"Xuefeng Bai","orcid":"https://orcid.org/0000-0001-7044-0683","observed_orcids":["https://orcid.org/0000-0001-7044-0683"]},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuefeng Bai","raw_affiliation_strings":["School of Computer Science and Technol- ogy , Harbin Institute of Technology , Shenzhen 518055 , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Technol- ogy , Harbin Institute of Technology , Shenzhen 518055 , China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5126522263","display_name":"Min Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Zhang","raw_affiliation_strings":["School of Computer Science and Technol- ogy , Harbin Institute of Technology , Shenzhen 518055 , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Technol- ogy , Harbin Institute of Technology , Shenzhen 518055 , China","institution_ids":["https://openalex.org/I204983213"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5123961460"],"corresponding_institution_ids":["https://openalex.org/I204983213"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.7411999702453613,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.7411999702453613,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10759","display_name":"Translation Studies and Practices","score":0.056699998676776886,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.03849999979138374,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/video-translation","display_name":"video translation","score":0.9980000257492065},{"id":"https://openalex.org/keywords/multimodal-large-language-models","display_name":"multimodal large language models","score":0.9760000109672546},{"id":"https://openalex.org/keywords/lip-synchronization","display_name":"lip synchronization","score":0.7599999904632568},{"id":"https://openalex.org/keywords/speech-synthesis","display_name":"speech synthesis","score":0.597000002861023},{"id":"https://openalex.org/keywords/temporal-reasoning","display_name":"temporal reasoning","score":0.6620000004768372},{"id":"https://openalex.org/keywords/multimodal-fusion","display_name":"multimodal fusion","score":0.8299999833106995},{"id":"https://openalex.org/keywords/visual-alignment","display_name":"visual alignment","score":0.5910000205039978},{"id":"https://openalex.org/keywords/zero-shot-translation","display_name":"zero-shot translation","score":0.7490000128746033}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8457000255584717},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5647000074386597},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.439300000667572},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.42320001125335693},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4221000075340271},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.40549999475479126},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.4027000069618225},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.4000999927520752},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.38940000534057617},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.38519999384880066},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.35740000009536743},{"id":"https://openalex.org/C540522513","wikidata":"https://www.wikidata.org/wiki/Q854954","display_name":"Videotelephony","level":2,"score":0.3375999927520752},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.32179999351501465},{"id":"https://openalex.org/C202474056","wikidata":"https://www.wikidata.org/wiki/Q1931635","display_name":"Video tracking","level":3,"score":0.299699991941452},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.2976999878883362},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.27399998903274536},{"id":"https://openalex.org/C65483669","wikidata":"https://www.wikidata.org/wiki/Q3536669","display_name":"Video processing","level":2,"score":0.26570001244544983},{"id":"https://openalex.org/C2780366754","wikidata":"https://www.wikidata.org/wiki/Q7494857","display_name":"Speech translation","level":3,"score":0.2556999921798706}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.36227/techrxiv.177162137.77131035/v1","doi":"https://doi.org/10.36227/techrxiv.177162137.77131035/v1","is_oa":true,"landing_page_url":"https://doi.org/10.36227/techrxiv.177162137.77131035/v1","pdf_url":"https://www.techrxiv.org/doi/pdf/10.36227/techrxiv.177162137.77131035","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.36227/techrxiv.177162137.77131035/v1","doi":"https://doi.org/10.36227/techrxiv.177162137.77131035/v1","is_oa":true,"landing_page_url":"https://doi.org/10.36227/techrxiv.177162137.77131035/v1","pdf_url":"https://www.techrxiv.org/doi/pdf/10.36227/techrxiv.177162137.77131035","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7130690807.pdf","grobid_xml":"https://content.openalex.org/works/W7130690807.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7154375666","https://openalex.org/W7106233266","https://openalex.org/W4410297058","https://openalex.org/W4410229152","https://openalex.org/W4417126502","https://openalex.org/W4405172761","https://openalex.org/W4410027467","https://openalex.org/W7152589936","https://openalex.org/W4413146669","https://openalex.org/W4394973308"],"abstract_inverted_index":{"Recent":[0],"developments":[1],"in":[2,73,198],"video":[3,11,34,111,129,144,173,199,214],"translation":[4,35,66,112],"have":[5],"further":[6,182],"enhanced":[7],"cross-lingual":[8],"access":[9],"to":[10,222],"content,":[12],"with":[13],"multimodal":[14,27,149,204],"large":[15],"language":[16],"models":[17],"(MLLMs)":[18],"playing":[19],"an":[20],"increasingly":[21],"important":[22],"supporting":[23],"role.":[24],"With":[25],"strong":[26],"understanding,":[28,101,145,200],"reasoning,":[29,147],"and":[30,54,75,85,95,104,148,157,165,178,187,203,206],"generation":[31],"capabilities,":[32],"MLLMsbased":[33],"systems":[36],"are":[37,220],"overcoming":[38],"the":[39,90,123],"limitations":[40],"of":[41,93,107,127],"traditional":[42],"cascaded":[43],"pipelines":[44],"that":[45],"separately":[46],"handle":[47],"automatic":[48],"speech":[49,163],"recognition,":[50],"machine":[51],"translation,":[52,130],"text-to-speech":[53],"lip":[55],"synchronization.":[56],"These":[57],"MLLMs-powered":[58],"approaches":[59],"not":[60],"only":[61],"achieve":[62],"competitive":[63],"or":[64],"superior":[65],"quality,":[67],"but":[68],"also":[69],"offer":[70],"better":[71],"robustness":[72],"zero-shot":[74],"multi-speaker,":[76],"while":[77],"jointly":[78],"modeling":[79],"semantic":[80],"fidelity,":[81],"timing,":[82],"speaker":[83],"identity,":[84],"emotional":[86],"consistency.":[87],"However,":[88],"despite":[89],"rapid":[91],"progress":[92],"MLLMs":[94,109,142],"extensive":[96],"surveys":[97],"on":[98],"general":[99],"video-language":[100],"a":[102,133],"focused":[103],"systematic":[105],"review":[106],"how":[108,141],"empower":[110],"tasks":[113],"is":[114],"still":[115],"lacking.":[116],"To":[117],"fill":[118],"this":[119],"gap,":[120],"we":[121,194],"provides":[122],"first":[124],"comprehensive":[125],"overview":[126],"MLLMs-based":[128],"organized":[131],"around":[132],"three-role":[134],"taxonomy:":[135],"1)":[136],"Semantic":[137],"Reasoner,":[138],"which":[139,154,169],"characterizes":[140],"perform":[143],"temporal":[146,201],"fusion;":[150],"2)":[151],"Expressive":[152],"Performer,":[153],"analyzes":[155],"LLM-driven":[156],"LLM-augmented":[158],"techniques":[159],"for":[160,175,190,212],"expressive,":[161],"controllable":[162],"generation;":[164],"3)":[166],"Visual":[167],"Synthesizer,":[168],"studies":[170],"different":[171],"based":[172],"generators":[174],"high-fidelity":[176],"lip-sync":[177],"visual":[179],"alignment.":[180],"We":[181],"summarize":[183],"representative":[184],"benchmarks,":[185],"datasets,":[186],"evaluation":[188],"metrics":[189],"these":[191],"roles.":[192],"Finally,":[193],"discuss":[195],"open":[196],"challenges":[197],"modeling,":[202],"alignment,":[205],"outline":[207],"promising":[208],"future":[209],"research":[210],"directions":[211],"MLLMspowered":[213],"translation.":[215],"For":[216],"more":[217],"details,":[218],"readers":[219],"encouraged":[221],"visit":[223],"our":[224],"repository":[225],"at":[226],"https://github.com/qbzz10/Video-Translation-using-MLLMs.":[227]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-02-21T00:00:00"}