{"id":"https://openalex.org/W7161155374","doi":"https://doi.org/10.48550/arxiv.2605.13831","title":"Training Long-Context Vision-Language Models Effectively with Generalization Beyond 128K Context","display_name":"Training Long-Context Vision-Language Models Effectively with Generalization Beyond 128K Context","publication_year":2026,"publication_date":"2026-05-13","ids":{"openalex":"https://openalex.org/W7161155374","doi":"https://doi.org/10.48550/arxiv.2605.13831"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.13831","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.13831","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.13831","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136146916","display_name":"Zhaowei Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zhaowei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102431101","display_name":"Lishu Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Lishu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136140288","display_name":"Haodong Duan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Duan, Haodong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136099102","display_name":"Weiwei Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Weiwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136101781","display_name":"Sijin Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Sijin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136162792","display_name":"Ji Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Ji","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136164036","display_name":"Shen Yan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yan, Shen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136107274","display_name":"Shuai Peng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Shuai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026307571","display_name":"Sihang Yuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Sihang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136131281","display_name":"Chaoyi Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Chaoyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136169783","display_name":"Yi Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Yi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136168938","display_name":"Yangqiu Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Yangqiu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9829999804496765,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9829999804496765,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0035000001080334187,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.00139999995008111,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6858999729156494},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.6567999720573425},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6312999725341797},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.588699996471405},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.5339999794960022},{"id":"https://openalex.org/keywords/core","display_name":"Core (optical fiber)","score":0.5015000104904175},{"id":"https://openalex.org/keywords/foundation","display_name":"Foundation (evidence)","score":0.3472999930381775}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7312999963760376},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6858999729156494},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.6567999720573425},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6312999725341797},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.609000027179718},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.588699996471405},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.5339999794960022},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5315999984741211},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.5015000104904175},{"id":"https://openalex.org/C2780966255","wikidata":"https://www.wikidata.org/wiki/Q5474306","display_name":"Foundation (evidence)","level":2,"score":0.3472999930381775},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.3465999960899353},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.3357999920845032},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3237999975681305},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.3197000026702881},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.27559998631477356},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.26840001344680786},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2587999999523163},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.2549000084400177}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.13831","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.13831","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.13831","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.13831","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.4255484640598297,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Long-context":[0],"modeling":[1],"is":[2,79],"becoming":[3],"a":[4,49,59,173,221],"core":[5],"capability":[6],"of":[7,52],"modern":[8],"large":[9],"vision-language":[10,232],"models":[11],"(LVLMs),":[12],"enabling":[13],"sustained":[14],"context":[15,66],"management":[16],"across":[17,116],"long-document":[18,71,77,140,178],"understanding,":[19],"video":[20],"analysis,":[21],"and":[22,39,119,137,183,189,211,225],"multi-turn":[23],"tool":[24],"use":[25],"in":[26],"agentic":[27],"workflows.":[28],"Yet":[29],"practical":[30,222],"training":[31,195],"recipes":[32],"remain":[33],"insufficiently":[34],"explored,":[35],"particularly":[36],"for":[37,56,98,134,154,229],"designing":[38],"balancing":[40],"long-context":[41,53,110,166,208,231],"data":[42,102,105,133,150],"mixtures.":[43],"In":[44],"this":[45,88],"work,":[46],"we":[47,161],"present":[48],"systematic":[50],"study":[51,219],"continued":[54,167],"pre-training":[55,168],"LVLMs,":[57],"extending":[58],"7B":[60],"model":[61],"from":[62,169],"32K":[63],"to":[64,203],"128K":[65,194],"with":[67,130,171],"extensive":[68],"ablations":[69,91],"on":[70,87,158],"data.":[72],"We":[73],"first":[74],"show":[75],"that":[76,109,147],"VQA":[78,141,179],"substantially":[80],"more":[81],"effective":[82],"than":[83],"OCR":[84],"transcription.":[85],"Building":[86],"observation,":[89],"our":[90,218],"further":[92,201],"yield":[93],"three":[94],"key":[95],"findings:":[96],"i)":[97],"sequence-length":[99],"distribution,":[100],"balanced":[101],"outperforms":[103],"target-length-focused":[104],"(e.g.,":[106],"128K),":[107],"suggesting":[108,146],"ability":[111],"requires":[112],"generalizable":[113],"key-information":[114],"retrieval":[115,122],"various":[117],"lengths":[118],"positions;":[120],"ii)":[121],"remains":[123],"the":[124,152],"primary":[125],"bottleneck,":[126],"favoring":[127],"retrieval-heavy":[128],"mixtures":[129],"modest":[131],"reasoning":[132],"task":[135],"diversity;":[136],"iii)":[138],"pure":[139],"largely":[142],"preserves":[143],"short-context":[144],"capabilities,":[145],"instruction-formatted":[148],"long":[149],"reduces":[151],"need":[153],"short-data":[155],"mixing.":[156],"Based":[157],"these":[159],"findings,":[160],"introduce":[162],"MMProLong,":[163],"obtained":[164],"by":[165,181],"Qwen2.5-VL-7B":[170],"only":[172],"5B-token":[174],"budget.":[175],"MMProLong":[176],"improves":[177],"scores":[180],"7.1%":[182],"maintains":[184],"strong":[185],"performance":[186],"at":[187],"256K":[188],"512K":[190],"contexts":[191],"beyond":[192],"its":[193],"window,":[196],"without":[197,214],"additional":[198],"training.":[199],"It":[200],"generalizes":[202],"webpage-based":[204],"multimodal":[205],"needle":[206],"retrieval,":[207],"vision-text":[209],"compression,":[210],"long-video":[212],"understanding":[213],"task-specific":[215],"supervision.":[216],"Overall,":[217],"establishes":[220],"LongPT":[223],"recipe":[224],"an":[226],"empirical":[227],"foundation":[228],"advancing":[230],"models.":[233]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-15T00:00:00"}