{"id":"https://openalex.org/W4389520299","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.715","title":"Orthogonal Subspace Learning for Language Model Continual Learning","display_name":"Orthogonal Subspace Learning for Language Model Continual Learning","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389520299","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.715"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.715","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.715","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.715","pdf_url":"https://aclanthology.org/2023.findings-emnlp.715.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.findings-emnlp.715.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Xiao Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiao Wang","raw_affiliation_strings":["\u22c6 School of Computer Science, Fudan University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u22c6 School of Computer Science, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046826993","display_name":"Tianze Chen","orcid":"https://orcid.org/0009-0004-9016-5294","observed_orcids":["https://orcid.org/0009-0004-9016-5294"]},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianze Chen","raw_affiliation_strings":["\u22c6 School of Computer Science, Fudan University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u22c6 School of Computer Science, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109786310","display_name":"Qiming Ge","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qiming Ge","raw_affiliation_strings":["\u22c6 School of Computer Science, Fudan University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u22c6 School of Computer Science, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101085871","display_name":"Xia Han","orcid":"https://orcid.org/0009-0000-1870-7491","observed_orcids":["https://orcid.org/0009-0000-1870-7491"]},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Han Xia","raw_affiliation_strings":["\u22c6 School of Computer Science, Fudan University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u22c6 School of Computer Science, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100615025","display_name":"Rong Bao","orcid":"https://orcid.org/0000-0002-0877-9093","observed_orcids":["https://orcid.org/0000-0002-0877-9093"]},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rong Bao","raw_affiliation_strings":["\u22c6 School of Computer Science, Fudan University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u22c6 School of Computer Science, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055879965","display_name":"Rui Zheng","orcid":"https://orcid.org/0000-0002-4225-7133","observed_orcids":["https://orcid.org/0000-0002-4225-7133","https://orcid.org/0009-0000-8602-617X"]},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rui Zheng","raw_affiliation_strings":["\u22c6 School of Computer Science, Fudan University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u22c6 School of Computer Science, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102230303","display_name":"Qi Zhang","orcid":"https://orcid.org/0009-0007-2552-087X","observed_orcids":["https://orcid.org/0009-0007-2552-087X","https://orcid.org/0000-0002-3094-282X"]},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qi Zhang","raw_affiliation_strings":["\u22c6 School of Computer Science, Fudan University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u22c6 School of Computer Science, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058353652","display_name":"Tao Gui","orcid":"https://orcid.org/0000-0002-6154-0751","observed_orcids":["https://orcid.org/0000-0002-6154-0751"]},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tao Gui","raw_affiliation_strings":["Institute of Modern Languages and Linguistics, Fudan University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Modern Languages and Linguistics, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5088834359","display_name":"Xuanjing Huang","orcid":"https://orcid.org/0000-0001-9197-9426","observed_orcids":["https://orcid.org/0000-0001-9197-9426"]},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuanjing Huang","raw_affiliation_strings":["International Human Phenome Institutes (Shanghai)","School of Computer Science , Fudan University , Shanghai , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"International Human Phenome Institutes (Shanghai)","institution_ids":[]},{"raw_affiliation_string":"School of Computer Science , Fudan University , Shanghai , China","institution_ids":["https://openalex.org/I24943067"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I24943067"],"apc_list":null,"apc_paid":null,"fwci":25.2749,"has_fulltext":true,"cited_by_count":228,"citation_normalized_percentile":{"value":0.99692517,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"10658","last_page":"10671"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.005400000140070915,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/catastrophic-forgetting","display_name":"catastrophic forgetting","score":0.9900000095367432},{"id":"https://openalex.org/keywords/low-rank-adaptation","display_name":"low-rank adaptation","score":0.9879999756813049},{"id":"https://openalex.org/keywords/task-interference","display_name":"task interference","score":0.4880000054836273},{"id":"https://openalex.org/keywords/parameter-efficient-fine-tuning","display_name":"parameter-efficient fine-tuning","score":0.7020000219345093},{"id":"https://openalex.org/keywords/generalization-to-unseen-tasks","display_name":"generalization to unseen tasks","score":0.8270000219345093}],"concepts":[{"id":"https://openalex.org/C7149132","wikidata":"https://www.wikidata.org/wiki/Q1377840","display_name":"Forgetting","level":2,"score":0.8150525689125061},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7740395665168762},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.5843371748924255},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5738843679428101},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5245756506919861},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5023791790008545},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.4891236424446106},{"id":"https://openalex.org/C12362212","wikidata":"https://www.wikidata.org/wiki/Q728435","display_name":"Linear subspace","level":2,"score":0.4399409592151642},{"id":"https://openalex.org/C32834561","wikidata":"https://www.wikidata.org/wiki/Q660730","display_name":"Subspace topology","level":2,"score":0.43689581751823425},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2023.findings-emnlp.715","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.715","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.715","pdf_url":"https://aclanthology.org/2023.findings-emnlp.715.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2310.14152","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.14152","pdf_url":"https://arxiv.org/pdf/2310.14152","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.14152","doi":"https://doi.org/10.48550/arxiv.2310.14152","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.14152","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.715","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.715","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.715","pdf_url":"https://aclanthology.org/2023.findings-emnlp.715.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320335796","display_name":"Program of Shanghai Academic Research Leader","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389520299.pdf","grobid_xml":"https://content.openalex.org/works/W4389520299.grobid-xml"},"referenced_works_count":47,"referenced_works":["https://openalex.org/W1682403713","https://openalex.org/W2113459411","https://openalex.org/W2170240176","https://openalex.org/W2473930607","https://openalex.org/W2560647685","https://openalex.org/W2734314755","https://openalex.org/W2799054028","https://openalex.org/W2902456977","https://openalex.org/W2902625698","https://openalex.org/W2913946806","https://openalex.org/W2923014074","https://openalex.org/W2926477959","https://openalex.org/W2948743095","https://openalex.org/W2963341956","https://openalex.org/W2980753060","https://openalex.org/W2981852735","https://openalex.org/W2990704537","https://openalex.org/W3030163527","https://openalex.org/W3035691277","https://openalex.org/W3083410900","https://openalex.org/W3121904249","https://openalex.org/W3123155632","https://openalex.org/W3152956381","https://openalex.org/W3168867926","https://openalex.org/W3171057731","https://openalex.org/W3176828726","https://openalex.org/W3205616434","https://openalex.org/W3205949070","https://openalex.org/W4223458030","https://openalex.org/W4226278401","https://openalex.org/W4285157946","https://openalex.org/W4287117014","https://openalex.org/W4306387799","https://openalex.org/W4309953708","https://openalex.org/W4312238419","https://openalex.org/W4318751794","https://openalex.org/W4322718191","https://openalex.org/W4327810158","https://openalex.org/W4365475020","https://openalex.org/W4366400290","https://openalex.org/W4376167151","https://openalex.org/W4385570539","https://openalex.org/W4385571224","https://openalex.org/W4385572845","https://openalex.org/W4387561459","https://openalex.org/W4392173735","https://openalex.org/W4406800520"],"related_works":["https://openalex.org/W4414756253","https://openalex.org/W7204987730","https://openalex.org/W7154704365","https://openalex.org/W7117689555","https://openalex.org/W4412944969","https://openalex.org/W6891937370","https://openalex.org/W4403813230","https://openalex.org/W7166799902","https://openalex.org/W4415334918","https://openalex.org/W7196977170"],"abstract_inverted_index":{"Benefiting":[0],"from":[1],"massive":[2],"corpora":[3],"and":[4,17,45,89],"advanced":[5],"hardware,":[6],"large":[7],"language":[8,15,52],"models":[9],"(LLMs)":[10],"exhibit":[11],"remarkable":[12],"capabilities":[13],"in":[14,22,51,65,77,116],"understanding":[16],"generation.However,":[18],"their":[19],"performance":[20],"degrades":[21],"scenarios":[23],"where":[24],"multiple":[25],"tasks":[26,64],"are":[27,71],"encountered":[28],"sequentially,":[29],"also":[30],"known":[31],"as":[32],"catastrophic":[33,56],"forgetting.In":[34],"this":[35],"paper,":[36],"we":[37],"propose":[38],"orthogonal":[39,73],"low-rank":[40],"adaptation":[41],"(O-LoRA),":[42],"a":[43],"simple":[44],"efficient":[46],"approach":[47],"for":[48,95],"continual":[49,99],"learning":[50,59,100],"models,":[53],"effectively":[54],"mitigating":[55],"forgetting":[57],"while":[58],"new":[60],"tasks.Specifically,":[61],"O-LoRA":[62],"learns":[63],"different":[66],"(low-rank)":[67],"vector":[68],"subspaces":[69],"that":[70,103],"kept":[72],"to":[74,79,110],"each":[75],"other":[76],"order":[78],"minimize":[80],"interference.Our":[81],"method":[82,105,114],"induces":[83],"only":[84],"marginal":[85],"additional":[86],"parameter":[87],"costs":[88],"requires":[90],"no":[91],"user":[92],"data":[93],"storage":[94],"replay.Experimental":[96],"results":[97],"on":[98,123],"benchmarks":[101],"show":[102],"our":[104,113],"outperforms":[106],"state-of-the-art":[107],"methods.Furthermore,":[108],"compared":[109],"previous":[111],"approaches,":[112],"excels":[115],"preserving":[117],"the":[118],"generalization":[119],"ability":[120],"of":[121],"LLMs":[122],"unseen":[124],"tasks.":[125]},"counts_by_year":[{"year":2026,"cited_by_count":122},{"year":2025,"cited_by_count":66},{"year":2024,"cited_by_count":40}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-12-11T00:00:00"}