{"id":"https://openalex.org/W4397028161","doi":"https://doi.org/10.48550/arxiv.2405.09673","title":"LoRA Learns Less and Forgets Less","display_name":"LoRA Learns Less and Forgets Less","publication_year":2024,"publication_date":"2024-05-15","ids":{"openalex":"https://openalex.org/W4397028161","doi":"https://doi.org/10.48550/arxiv.2405.09673"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2405.09673","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.09673","pdf_url":"https://arxiv.org/pdf/2405.09673","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2405.09673","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071084322","display_name":"Dan Biderman","orcid":"https://orcid.org/0000-0003-2054-8439"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Biderman, Dan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098684031","display_name":"Jose Gonzalez Ortiz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Portes, Jacob","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051310825","display_name":"Jacob Portes","orcid":"https://orcid.org/0000-0003-3102-012X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ortiz, Jose Javier Gonzalez","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036307641","display_name":"Mansheej Paul","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paul, Mansheej","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098684032","display_name":"Philip Greengard","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Greengard, Philip","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098684033","display_name":"Connor Jennings","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jennings, Connor","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067437521","display_name":"Daniel King","orcid":"https://orcid.org/0000-0002-6678-0586"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"King, Daniel","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109658192","display_name":"Sam Havens","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Havens, Sam","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098684034","display_name":"Vitaliy Chiley","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chiley, Vitaliy","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078716102","display_name":"Jonathan Frankle","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Frankle, Jonathan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002287514","display_name":"Cody Blakeney","orcid":"https://orcid.org/0000-0002-1412-2813"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Blakeney, Cody","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5030697996","display_name":"John P. Cunningham","orcid":"https://orcid.org/0000-0001-9125-3027"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cunningham, John P.","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":12,"corresponding_author_ids":["https://openalex.org/A5071084322"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.3912999927997589,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.3912999927997589,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4862271845340729},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.41066649556159973},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.34184616804122925}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4862271845340729},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.41066649556159973},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.34184616804122925}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2405.09673","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.09673","pdf_url":"https://arxiv.org/pdf/2405.09673","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"doi:10.48550/arxiv.2405.09673","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2405.09673","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2405.09673","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.09673","pdf_url":"https://arxiv.org/pdf/2405.09673","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W4391375266","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"Low-Rank":[0],"Adaptation":[1],"(LoRA)":[2],"is":[3,127],"a":[4,124],"widely-used":[5],"parameter-efficient":[6],"finetuning":[7,37,50,120,148],"method":[8],"for":[9,147],"large":[10],"language":[11],"models.":[12],"LoRA":[13,34,72,78,94,132],"saves":[14],"memory":[15],"by":[16,143],"training":[17],"only":[18],"low":[19],"rank":[20,125],"perturbations":[21,122],"to":[22],"selected":[23],"weight":[24,104],"matrices.":[25],"In":[26],"this":[27],"work,":[28],"we":[29,116],"compare":[30],"the":[31,48,68,81,88,138],"performance":[32,84],"of":[33,137],"and":[35,43,55,106],"full":[36,75,119],"on":[38,85],"two":[39],"target":[40,89],"domains,":[41],"programming":[42],"mathematics.":[44],"We":[45,91,141],"consider":[46],"both":[47],"instruction":[49],"(approximately":[51],"100K":[52],"prompt-response":[53],"pairs)":[54],"continued":[56],"pretraining":[57],"(20B":[58],"unstructured":[59],"tokens)":[60],"data":[61],"regimes.":[62],"Our":[63],"results":[64],"show":[65,92,117],"that,":[66],"in":[67],"standard":[69],"low-rank":[70],"settings,":[71],"substantially":[73],"underperforms":[74],"finetuning.":[76],"Nevertheless,":[77],"better":[79],"maintains":[80],"base":[82],"model's":[83],"tasks":[86],"outside":[87],"domain.":[90],"that":[93,118,126],"mitigates":[95],"forgetting":[96],"more":[97,112],"than":[98,130],"common":[99],"regularization":[100],"techniques":[101],"such":[102],"as":[103],"decay":[105],"dropout;":[107],"it":[108],"also":[109],"helps":[110],"maintain":[111],"diverse":[113],"generations.":[114],"Finally,":[115],"learns":[121],"with":[123,149],"10-100X":[128],"greater":[129],"typical":[131],"configurations,":[133],"possibly":[134],"explaining":[135],"some":[136],"reported":[139],"gaps.":[140],"conclude":[142],"proposing":[144],"best":[145],"practices":[146],"LoRA.":[150]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-03-10T16:38:18.471706","created_date":"2025-10-10T00:00:00"}