{"id":"https://openalex.org/W4412945071","doi":"https://doi.org/10.18653/v1/2025.acl-long.813","title":"Velocitune: A Velocity-based Dynamic Domain Reweighting Method for Continual Pre-training","display_name":"Velocitune: A Velocity-based Dynamic Domain Reweighting Method for Continual Pre-training","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412945071","doi":"https://doi.org/10.18653/v1/2025.acl-long.813"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.acl-long.813","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.813","pdf_url":"https://aclanthology.org/2025.acl-long.813.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.acl-long.813.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075367840","display_name":"Zheheng Luo","orcid":"https://orcid.org/0000-0001-8246-5511"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheheng Luo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100327606","display_name":"Xin Zhang","orcid":"https://orcid.org/0000-0003-3467-6132"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xin Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115596171","display_name":"Xiao Liu","orcid":"https://orcid.org/0000-0002-8029-0217"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101349636","display_name":"Haoling Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haoling Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111237866","display_name":"Yeyun Gong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yeyun Gong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100340225","display_name":"Qi Chen","orcid":"https://orcid.org/0009-0009-8193-2697"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5054004904","display_name":"Peng Cheng","orcid":"https://orcid.org/0009-0006-6928-3012"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng Cheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"16644","last_page":"16656"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11398","display_name":"Hand Gesture Recognition Systems","score":0.8191999793052673,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11398","display_name":"Hand Gesture Recognition Systems","score":0.8191999793052673,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.7508999705314636,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12537","display_name":"Flow Measurement and Analysis","score":0.7263000011444092,"subfield":{"id":"https://openalex.org/subfields/2211","display_name":"Mechanics of Materials"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6821276545524597},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.6343586444854736},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.4823307693004608},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3676486611366272},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14943352341651917},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.09531262516975403},{"id":"https://openalex.org/keywords/mathematical-analysis","display_name":"Mathematical analysis","score":0.054292112588882446}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6821276545524597},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6343586444854736},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.4823307693004608},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3676486611366272},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14943352341651917},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.09531262516975403},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.054292112588882446},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.acl-long.813","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.813","pdf_url":"https://aclanthology.org/2025.acl-long.813.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.acl-long.813","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.813","pdf_url":"https://aclanthology.org/2025.acl-long.813.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412945071.pdf","grobid_xml":"https://content.openalex.org/works/W4412945071.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W230091440","https://openalex.org/W2390279801","https://openalex.org/W2233261550","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2810751659"],"abstract_inverted_index":{"It":[0],"is":[1,7,88],"well-known":[2],"that":[3,68,154],"a":[4,19,65,91,104,117,129],"diverse":[5],"corpus":[6,130],"critical":[8],"for":[9,100],"training":[10,31,45],"large":[11],"language":[12],"models,":[13],"which":[14,87],"are":[15],"typically":[16],"constructed":[17],"from":[18,33],"mixture":[20],"of":[21,55,111,131,160],"various":[22],"domains.In":[23],"general,":[24],"previous":[25],"efforts":[26],"resort":[27],"to":[28,46,94],"either":[29],"sampling":[30],"data":[32,75,166],"different":[34],"domains":[35,81],"with":[36,103,123],"static":[37],"proportions":[38,43,76],"or":[39],"dynamically":[40,69],"adjusting":[41],"these":[42],"during":[44],"optimise":[47],"pretraining":[48],"performance.However,":[49],"few":[50],"methods":[51],"addressed":[52],"the":[53,96,109,158],"complexity":[54],"domain-adaptive":[56],"continual":[57],"pre-training.To":[58],"fill":[59],"this":[60],"gap,":[61],"we":[62,113],"propose":[63],"Velocitune,":[64,112],"novel":[66],"framework":[67],"assesses":[70],"learning":[71,80,85,98],"velocity":[72],"and":[73,136,144,148,165],"adjusts":[74],"accordingly,":[77],"favouring":[78],"slower":[79],"while":[82],"de-emphasising":[83],"faster":[84],"ones,":[86],"guided":[89],"by":[90],"scaling":[92],"law":[93],"estimate":[95],"desired":[97],"goal":[99],"each":[101],"domain":[102],"less":[105],"associated":[106],"cost.To":[107],"evaluate":[108],"effectiveness":[110,159],"conduct":[114],"experiments":[115],"on":[116,120,128],"dataset":[118],"focused":[119],"reasoning":[121,146],"tasks":[122,147],"CodeLlama,":[124],"as":[125,127],"well":[126],"system":[132],"commands":[133],"using":[134],"Llama3":[135],"Mistral.Velocitune":[137],"achieves":[138],"performance":[139],"gains":[140],"in":[141],"both":[142],"math":[143],"code":[145],"command-line":[149],"generation":[150],"benchmarks.Further":[151],"analysis":[152],"reveals":[153],"key":[155],"factors":[156],"driving":[157],"Velocitune":[161],"include":[162],"target":[163],"estimation":[164],"ordering.":[167]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}