{"id":"https://openalex.org/W4391013432","doi":"https://doi.org/10.48550/arxiv.2401.09181","title":"Beyond Anti-Forgetting: Multimodal Continual Instruction Tuning with Positive Forward Transfer","display_name":"Beyond Anti-Forgetting: Multimodal Continual Instruction Tuning with Positive Forward Transfer","publication_year":2024,"publication_date":"2024-01-17","ids":{"openalex":"https://openalex.org/W4391013432","doi":"https://doi.org/10.48550/arxiv.2401.09181"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2401.09181","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.09181","pdf_url":"https://arxiv.org/pdf/2401.09181","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2401.09181","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101773557","display_name":"Junhao Zheng","orcid":"https://orcid.org/0000-0001-9124-2467"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Zheng, Junhao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076609933","display_name":"Qianli Ma","orcid":"https://orcid.org/0000-0002-9356-2883"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Qianli","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003356618","display_name":"Zhen Liu","orcid":"https://orcid.org/0000-0002-8107-0929"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Zhen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079609030","display_name":"Binquan Wu","orcid":"https://orcid.org/0009-0008-5031-8339"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Binquan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5104245194","display_name":"Huawen Feng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feng, Huawen","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5101773557"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9840999841690063,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9828000068664551,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/forgetting","display_name":"Forgetting","score":0.8524227142333984},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7356899976730347},{"id":"https://openalex.org/keywords/retraining","display_name":"Retraining","score":0.5452386736869812},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.46845221519470215},{"id":"https://openalex.org/keywords/subspace-topology","display_name":"Subspace topology","score":0.4558213949203491},{"id":"https://openalex.org/keywords/transfer-of-learning","display_name":"Transfer of learning","score":0.43466565012931824},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.35135066509246826},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.11682116985321045},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.06581801176071167}],"concepts":[{"id":"https://openalex.org/C7149132","wikidata":"https://www.wikidata.org/wiki/Q1377840","display_name":"Forgetting","level":2,"score":0.8524227142333984},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7356899976730347},{"id":"https://openalex.org/C2778712577","wikidata":"https://www.wikidata.org/wiki/Q3505966","display_name":"Retraining","level":2,"score":0.5452386736869812},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46845221519470215},{"id":"https://openalex.org/C32834561","wikidata":"https://www.wikidata.org/wiki/Q660730","display_name":"Subspace topology","level":2,"score":0.4558213949203491},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.43466565012931824},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35135066509246826},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.11682116985321045},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.06581801176071167},{"id":"https://openalex.org/C155202549","wikidata":"https://www.wikidata.org/wiki/Q178803","display_name":"International trade","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2401.09181","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.09181","pdf_url":"https://arxiv.org/pdf/2401.09181","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2401.09181","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2401.09181","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2401.09181","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.09181","pdf_url":"https://arxiv.org/pdf/2401.09181","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391013432.pdf","grobid_xml":"https://content.openalex.org/works/W4391013432.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4289718052","https://openalex.org/W2164121020","https://openalex.org/W2081982437","https://openalex.org/W2145559838","https://openalex.org/W2027050655","https://openalex.org/W2905319430","https://openalex.org/W3116498279","https://openalex.org/W3028244590","https://openalex.org/W4254349500","https://openalex.org/W4287549553"],"abstract_inverted_index":{"Multimodal":[0,6],"Continual":[1],"Instruction":[2],"Tuning":[3,105],"(MCIT)":[4],"enables":[5],"Large":[7],"Language":[8],"Models":[9],"(MLLMs)":[10],"to":[11,91,119,123,129,163,175],"meet":[12],"continuously":[13,160],"emerging":[14],"requirements":[15],"without":[16],"expensive":[17],"retraining.":[18],"MCIT":[19],"faces":[20],"two":[21],"major":[22],"obstacles:":[23],"catastrophic":[24,49,92],"forgetting":[25,93],"(where":[26,35],"old":[27,86,152],"knowledge":[28],"is":[29,41],"forgotten)":[30],"and":[31,87,94,128,149,171],"negative":[32,55,95],"forward":[33,56,96],"transfer":[34],"the":[36,80,116,120,130,157,167],"performance":[37,144],"of":[38,159],"future":[39,173],"tasks":[40,127,165],"degraded).":[42],"Although":[43],"existing":[44],"methods":[45],"have":[46],"greatly":[47],"alleviated":[48],"forgetting,":[50],"they":[51],"still":[52],"suffer":[53],"from":[54],"transfer.":[57,97],"We":[58],"discover":[59],"a":[60,111],"large":[61],"discrepancy":[62,77],"in":[63,79],"different":[64],"input":[65,74],"embeddings":[66],"by":[67],"performing":[68],"singular":[69],"value":[70],"decomposition":[71],"(SVD)":[72],"on":[73],"embeddings.":[75],"This":[76],"results":[78],"model":[81],"learning":[82],"irrelevant":[83],"information":[84],"for":[85,133],"pre-trained":[88,131,135],"tasks,":[89],"leading":[90],"To":[98],"address":[99],"these":[100],"issues,":[101],"we":[102],"propose":[103],"Prompt":[104],"with":[106],"Positive":[107],"Forward":[108],"Transfer":[109],"(Fwd-Prompt),":[110],"prompt-based":[112],"method":[113],"that":[114,140],"projects":[115],"prompt":[117],"gradient":[118],"residual":[121],"space":[122],"minimize":[124],"interference":[125],"between":[126],"subspace":[132],"reusing":[134],"knowledge.":[136],"Our":[137,154],"experiments":[138],"demonstrate":[139],"Fwd-Prompt":[141],"achieves":[142],"state-of-the-art":[143],"while":[145],"updating":[146],"fewer":[147],"parameters":[148],"requiring":[150],"no":[151],"samples.":[153],"research":[155],"illuminates":[156],"potential":[158],"adapting":[161],"MLLMs":[162],"new":[164],"under":[166],"instruction":[168],"tuning":[169],"paradigm":[170],"encourages":[172],"studies":[174],"explore":[176],"MCIT.":[177]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-03-07T16:01:11.037858","created_date":"2024-01-19T00:00:00"}