{"id":"https://openalex.org/W4412889571","doi":"https://doi.org/10.18653/v1/2025.acl-long.1578","title":"Efficient Domain Continual pretraining by Mitigating the Stability Gap","display_name":"Efficient Domain Continual pretraining by Mitigating the Stability Gap","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412889571","doi":"https://doi.org/10.18653/v1/2025.acl-long.1578"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.acl-long.1578","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.1578","pdf_url":"https://aclanthology.org/2025.acl-long.1578.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.acl-long.1578.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044941285","display_name":"Yiduo Guo","orcid":"https://orcid.org/0000-0002-6323-6235"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yiduo Guo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101541695","display_name":"Jie Fu","orcid":"https://orcid.org/0000-0002-2593-995X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jie Fu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066928490","display_name":"Huishuai Zhang","orcid":"https://orcid.org/0000-0003-2711-7295"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huishuai Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5037132097","display_name":"Dongyan Zhao","orcid":"https://orcid.org/0000-0002-0396-6703"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dongyan Zhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"32850","last_page":"32870"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14156","display_name":"Engineering Applied Research","score":0.6970000267028809,"subfield":{"id":"https://openalex.org/subfields/2205","display_name":"Civil and Structural Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14156","display_name":"Engineering Applied Research","score":0.6970000267028809,"subfield":{"id":"https://openalex.org/subfields/2205","display_name":"Civil and Structural Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12809","display_name":"Contact Mechanics and Variational Inequalities","score":0.654699981212616,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11799","display_name":"Adhesion, Friction, and Surface Interactions","score":0.5892000198364258,"subfield":{"id":"https://openalex.org/subfields/2211","display_name":"Mechanics of Materials"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5566763877868652},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5493908524513245},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.5436389446258545},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.1333254873752594},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1293843388557434}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5566763877868652},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5493908524513245},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.5436389446258545},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.1333254873752594},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1293843388557434},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.acl-long.1578","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.1578","pdf_url":"https://aclanthology.org/2025.acl-long.1578.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.acl-long.1578","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.1578","pdf_url":"https://aclanthology.org/2025.acl-long.1578.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Climate action","score":0.4399999976158142,"id":"https://metadata.un.org/sdg/13"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412889571.pdf","grobid_xml":"https://content.openalex.org/works/W4412889571.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Continual":[0],"pretraining":[1,37,136],"enables":[2],"Large":[3],"Language":[4],"Models":[5],"(LLMs)":[6],"to":[7,9,127,158,181],"adapt":[8],"specialized":[10],"domains":[11],"like":[12],"medicine":[13],"and":[14,25,87,119,137,142,184,202],"law.However,":[15],"we":[16,52,96,177],"observe":[17],"a":[18,27,41,46,79,92,98],"consistent":[19],"phenomenon":[20],"across":[21],"different":[22],"model":[23,155],"sizes":[24],"domains:":[26],"temporary":[28],"performance":[29,42,90,151,173,198],"drop":[30,66],"at":[31],"the":[32,35,54,60,71,106,129,147,153,164,186,195],"start":[33],"of":[34,49,81,108,131,152,163],"continual":[36,135],"process,":[38],"followed":[39],"by":[40,104],"recovery":[43],"phase.To":[44],"gain":[45],"deeper":[47],"understanding":[48],"this":[50,64,76,84],"issue,":[51],"use":[53],"stability":[55],"gap-a":[56],"concept":[57],"adapted":[58],"from":[59,68,156],"visual":[61],"domain-which":[62],"explains":[63],"initial":[65,85],"arises":[67],"instability":[69,86,103],"in":[70,140],"model's":[72],"general":[73,171],"abilities.We":[74],"validate":[75,128],"hypothesis":[77],"through":[78],"series":[80],"experiments.To":[82],"address":[83],"enhance":[88],"LLM":[89],"within":[91],"fixed":[93],"compute":[94],"budget,":[95,167],"propose":[97],"training":[99,166],"strategy":[100],"that":[101],"mitigates":[102],"increasing":[105],"number":[107],"epochs,":[109],"alongside":[110],"two":[111],"data":[112,116],"sampling":[113],"strategies":[114,133,145,180],"targeting":[115],"domain":[117],"relevance":[118],"corpus":[120],"distribution.We":[121],"conduct":[122],"experiments":[123],"on":[124,205],"Llamafamily":[125],"models":[126,201],"effectiveness":[130],"our":[132,179],"for":[134],"instruction":[138],"tuning":[139],"medical":[141,149,197],"legal":[143],"domains.Our":[144],"improve":[146],"average":[148],"task":[150,172],"OpenLlama-3B":[154],"36.2%":[157],"40.7%":[159],"using":[160],"only":[161],"40%":[162],"original":[165],"while":[168],"also":[169],"enhancing":[170],"without":[174],"causing":[175],"forgetting.Furthermore,":[176],"apply":[178],"continually":[182],"pre-train":[183],"instruction-tune":[185],"Llama-3-8B":[187],"model.The":[188],"resulting":[189],"model,":[190],"Llama-3-Physician":[191],"1":[192],",":[193],"achieves":[194],"best":[196],"among":[199],"open-source":[200],"rivals":[203],"GPT-4":[204],"specific":[206],"tasks.":[207]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}