{"id":"https://openalex.org/W4405778808","doi":"https://doi.org/10.1109/tmm.2024.3521729","title":"VLAB: Enhancing Video Language Pretraining by Feature Adapting and Blending","display_name":"VLAB: Enhancing Video Language Pretraining by Feature Adapting and Blending","publication_year":2024,"publication_date":"2024-12-25","ids":{"openalex":"https://openalex.org/W4405778808","doi":"https://doi.org/10.1109/tmm.2024.3521729"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2024.3521729","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2024.3521729","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101777772","display_name":"Xingjian He","orcid":"https://orcid.org/0000-0001-5396-6253"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xingjian He","raw_affiliation_strings":["Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5396-6253","affiliations":[{"raw_affiliation_string":"Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210094879"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101953192","display_name":"Sihan Chen","orcid":"https://orcid.org/0000-0002-9453-6265"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sihan Chen","raw_affiliation_strings":["Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210094879"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017393243","display_name":"Fan Ma","orcid":"https://orcid.org/0000-0002-4131-1222"},"institutions":[{"id":"https://openalex.org/I168879160","display_name":"Zhejiang University of Science and Technology","ror":"https://ror.org/05mx0wr29","country_code":"CN","type":"education","lineage":["https://openalex.org/I168879160"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fan Ma","raw_affiliation_strings":["College of Computer Science and Technology, Zhejiang University, Zhejiang, China"],"raw_orcid":"https://orcid.org/0000-0002-4131-1222","affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Zhejiang, China","institution_ids":["https://openalex.org/I168879160"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012273973","display_name":"Zhicheng Huang","orcid":"https://orcid.org/0000-0003-0399-9668"},"institutions":[{"id":"https://openalex.org/I92403157","display_name":"University of Science and Technology Beijing","ror":"https://ror.org/02egmk993","country_code":"CN","type":"education","lineage":["https://openalex.org/I92403157"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhicheng Huang","raw_affiliation_strings":["University of Science and Technology Beijing, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology Beijing, Beijing, China","institution_ids":["https://openalex.org/I92403157"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100668699","display_name":"Xiaojie Jin","orcid":"https://orcid.org/0000-0002-7850-1353"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiaojie Jin","raw_affiliation_strings":["Bytedance Inc., Beijing, China","Bytedance Inc"],"raw_orcid":"https://orcid.org/0000-0002-7850-1353","affiliations":[{"raw_affiliation_string":"Bytedance Inc., Beijing, China","institution_ids":[]},{"raw_affiliation_string":"Bytedance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008547683","display_name":"Zikang Liu","orcid":"https://orcid.org/0000-0002-8947-7053"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zikang Liu","raw_affiliation_strings":["Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210094879"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016681542","display_name":"Dongmei Fu","orcid":"https://orcid.org/0000-0003-3918-9448"},"institutions":[{"id":"https://openalex.org/I92403157","display_name":"University of Science and Technology Beijing","ror":"https://ror.org/02egmk993","country_code":"CN","type":"education","lineage":["https://openalex.org/I92403157"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongmei Fu","raw_affiliation_strings":["University of Science and Technology Beijing, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3918-9448","affiliations":[{"raw_affiliation_string":"University of Science and Technology Beijing, Beijing, China","institution_ids":["https://openalex.org/I92403157"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005421447","display_name":"Yi Yang","orcid":"https://orcid.org/0000-0002-0512-880X"},"institutions":[{"id":"https://openalex.org/I168879160","display_name":"Zhejiang University of Science and Technology","ror":"https://ror.org/05mx0wr29","country_code":"CN","type":"education","lineage":["https://openalex.org/I168879160"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yi Yang","raw_affiliation_strings":["College of Computer Science and Technology, Zhejiang University, Zhejiang, China"],"raw_orcid":"https://orcid.org/0000-0002-0512-880X","affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Zhejiang, China","institution_ids":["https://openalex.org/I168879160"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108392430","display_name":"Jing Liu","orcid":"https://orcid.org/0000-0003-0903-9131"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jing Liu","raw_affiliation_strings":["Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0903-9131","affiliations":[{"raw_affiliation_string":"Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210094879"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100668696","display_name":"Jiashi Feng","orcid":"https://orcid.org/0000-0001-6843-0064"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiashi Feng","raw_affiliation_strings":["Bytedance Inc., Beijing, China","Bytedance Inc"],"raw_orcid":"https://orcid.org/0000-0001-6843-0064","affiliations":[{"raw_affiliation_string":"Bytedance Inc., Beijing, China","institution_ids":[]},{"raw_affiliation_string":"Bytedance Inc","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.1356,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.78332749,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"27","issue":null,"first_page":"2168","last_page":"2180"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9785000085830688,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9775000214576721,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8485822081565857},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.6214044690132141},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.6117360591888428},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.5021002292633057},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5006420612335205},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4625508189201355},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.37584495544433594},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.34813833236694336},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.336980402469635},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.09696745872497559}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8485822081565857},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.6214044690132141},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6117360591888428},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.5021002292633057},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5006420612335205},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4625508189201355},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.37584495544433594},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.34813833236694336},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.336980402469635},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.09696745872497559},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2024.3521729","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2024.3521729","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.5899999737739563,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1283858192","display_name":null,"funder_award_id":"U21B2043","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1508456709","display_name":null,"funder_award_id":"62102416","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":79,"referenced_works":["https://openalex.org/W1956340063","https://openalex.org/W2277195237","https://openalex.org/W2425121537","https://openalex.org/W2606982687","https://openalex.org/W2765716052","https://openalex.org/W2886641317","https://openalex.org/W2943885184","https://openalex.org/W2952132648","https://openalex.org/W2962945654","https://openalex.org/W2963017553","https://openalex.org/W2963341956","https://openalex.org/W3005881764","https://openalex.org/W3035635319","https://openalex.org/W3087975588","https://openalex.org/W3091588028","https://openalex.org/W3105232955","https://openalex.org/W3173223111","https://openalex.org/W3176641147","https://openalex.org/W3204588463","https://openalex.org/W3204670646","https://openalex.org/W3205021045","https://openalex.org/W3217340782","https://openalex.org/W4285606530","https://openalex.org/W4304014690","https://openalex.org/W4312299780","https://openalex.org/W4312463400","https://openalex.org/W4312884055","https://openalex.org/W4313136445","https://openalex.org/W4313186260","https://openalex.org/W4316661142","https://openalex.org/W4365385796","https://openalex.org/W4386071687","https://openalex.org/W4386076010","https://openalex.org/W4386076522","https://openalex.org/W4386076661","https://openalex.org/W4386083024","https://openalex.org/W4390871765","https://openalex.org/W4403511263","https://openalex.org/W4404003099","https://openalex.org/W6639432524","https://openalex.org/W6676497082","https://openalex.org/W6678262379","https://openalex.org/W6682631176","https://openalex.org/W6738045163","https://openalex.org/W6755207826","https://openalex.org/W6761014454","https://openalex.org/W6761197245","https://openalex.org/W6762122294","https://openalex.org/W6763228578","https://openalex.org/W6790019176","https://openalex.org/W6791353385","https://openalex.org/W6796581206","https://openalex.org/W6797109355","https://openalex.org/W6797148833","https://openalex.org/W6800139874","https://openalex.org/W6801013943","https://openalex.org/W6802744804","https://openalex.org/W6803545775","https://openalex.org/W6805349323","https://openalex.org/W6810042059","https://openalex.org/W6810334672","https://openalex.org/W6811013733","https://openalex.org/W6811072154","https://openalex.org/W6838434436","https://openalex.org/W6838701581","https://openalex.org/W6838756091","https://openalex.org/W6839015040","https://openalex.org/W6839670772","https://openalex.org/W6839948470","https://openalex.org/W6841325665","https://openalex.org/W6843018836","https://openalex.org/W6843263998","https://openalex.org/W6846313647","https://openalex.org/W6846867676","https://openalex.org/W6849177959","https://openalex.org/W6849307739","https://openalex.org/W6864544085","https://openalex.org/W6891781542","https://openalex.org/W6898505805"],"related_works":["https://openalex.org/W230091440","https://openalex.org/W2233261550","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2997094352","https://openalex.org/W3216976533","https://openalex.org/W100620283","https://openalex.org/W2495260952","https://openalex.org/W4366179611","https://openalex.org/W2996078371"],"abstract_inverted_index":{"Large-scale":[0],"image-text":[1],"contrastive":[2,112],"pre-training":[3,44,48,58],"models,":[4],"such":[5],"as":[6],"CLIP,":[7],"have":[8],"been":[9],"demonstrated":[10],"to":[11,56,96,109],"effectively":[12],"learn":[13],"high-quality":[14],"multimodal":[15,29,64,155],"representations.":[16],"However,":[17],"there":[18],"is":[19,75],"limited":[20],"research":[21],"on":[22,32,77,151,181],"learning":[23],"video-text":[24,43,71],"representations":[25,55],"for":[26,66],"general":[27],"video":[28,57,63,93,138,154,158,161,164,178],"tasks":[30,59],"based":[31],"these":[33],"powerful":[34],"features.":[35,139],"Towards":[36],"this":[37],"goal,":[38],"we":[39,89,119],"propose":[40,120],"a":[41,67,91],"novel":[42],"method":[45,124],"dubbed":[46],"VLAB:VideoLanguage":[47],"by":[49,131],"featureAdapting":[50],"andBlending,":[51],"which":[52],"transfers":[53],"CLIP":[54],"and":[60,83,104,113,137,144,163,173,184,194],"develops":[61],"unified":[62],"models":[65],"wide":[68],"range":[69],"of":[70,135,146,191],"tasks.":[72,115],"Specifically,":[73],"VLAB":[74,147,168],"founded":[76],"two":[78],"key":[79],"strategies:":[80],"feature":[81,84],"adapting":[82],"blending.":[85],"In":[86,116],"the":[87,106,117,128,133,142],"former,":[88],"introduce":[90],"new":[92,175],"adapter":[94],"module":[95],"address":[97],"CLIP's":[98],"deficiency":[99],"in":[100,177],"modeling":[101],"temporal":[102],"information":[103],"extend":[105],"model's":[107,129],"capability":[108],"encompass":[110],"both":[111],"generative":[114],"latter,":[118],"an":[121,189],"end-to-end":[122],"training":[123],"that":[125],"further":[126],"enhances":[127],"performance":[130],"exploiting":[132],"complementarity":[134],"image":[136],"We":[140],"validate":[141],"effectiveness":[143],"versatility":[145],"through":[148],"extensive":[149],"experiments":[150],"highly":[152],"competitive":[153],"tasks,":[156],"including":[157],"text":[159],"retrieval,":[160],"captioning,":[162],"question":[165,179],"answering.":[166],"Remarkably,":[167],"outperforms":[169],"competing":[170],"methods":[171],"significantly":[172],"sets":[174],"records":[176],"answering":[180],"MSRVTT,":[182],"MSVD,":[183],"TGIF":[185],"datasets.":[186],"It":[187],"achieves":[188],"accuracy":[190],"49.6,":[192],"60.9,":[193],"79.0,":[195],"respectively.":[196]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}