{"id":"https://openalex.org/W4393336136","doi":"https://doi.org/10.48550/arxiv.2403.19390","title":"Checkpoint Merging via Bayesian Optimization in LLM Pretraining","display_name":"Checkpoint Merging via Bayesian Optimization in LLM Pretraining","alternate_titles":[],"publication_year":2024,"publication_date":"2024-03-28","ids":{"openalex":"https://openalex.org/W4393336136","doi":"https://doi.org/10.48550/arxiv.2403.19390"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2403.19390","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.19390","pdf_url":"https://arxiv.org/pdf/2403.19390","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4393336136"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2403.19390","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049282010","display_name":"Deyuan Liu","orcid":"https://orcid.org/0000-0001-5764-8754","observed_orcids":["https://orcid.org/0000-0001-5764-8754"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Liu, Deyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089262598","display_name":"Zecheng Wang","orcid":"https://orcid.org/0000-0002-2282-3257","observed_orcids":["https://orcid.org/0000-0002-2282-3257"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Wang, Zecheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067986384","display_name":"Bingning Wang","orcid":"https://orcid.org/0009-0007-7748-7098","observed_orcids":["https://orcid.org/0009-0007-7748-7098","https://orcid.org/0000-0003-4095-5082"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Wang, Bingning","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115053594","display_name":"Weipeng Chen","orcid":"https://orcid.org/0000-0001-6396-9300","observed_orcids":["https://orcid.org/0000-0001-6396-9300"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Chen, Weipeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015377325","display_name":"Chunshan Li","orcid":"https://orcid.org/0000-0002-8419-0109","observed_orcids":["https://orcid.org/0000-0002-8419-0109"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Li, Chunshan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010541683","display_name":"Zhiying Tu","orcid":"https://orcid.org/0000-0001-8800-4513","observed_orcids":["https://orcid.org/0000-0001-8800-4513"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Tu, Zhiying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089932171","display_name":"Dianhui Chu","orcid":"https://orcid.org/0000-0001-8039-8394","observed_orcids":["https://orcid.org/0000-0001-8039-8394","https://orcid.org/0000-0003-2973-7252","https://orcid.org/0000-0002-2810-5264"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Chu, Dianhui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100725692","display_name":"Bo Li","orcid":"https://orcid.org/0000-0002-7113-284X","observed_orcids":["https://orcid.org/0000-0002-7113-284X"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Li, Bo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5031774490","display_name":"Dianbo Sui","orcid":"https://orcid.org/0000-0002-5200-2265","observed_orcids":["https://orcid.org/0000-0002-5200-2265"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Sui, Dianbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":46,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9101999998092651,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9101999998092651,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.04859999939799309,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.014100000262260437,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-pretraining","display_name":"large language model pretraining","score":0.921999990940094},{"id":"https://openalex.org/keywords/bayesian-optimization","display_name":"Bayesian optimization","score":0.9980000257492065},{"id":"https://openalex.org/keywords/training-trajectories","display_name":"training trajectories","score":0.3479999899864197},{"id":"https://openalex.org/keywords/training-cost-reduction","display_name":"training cost reduction","score":0.6800000071525574},{"id":"https://openalex.org/keywords/domain-generalization","display_name":"domain generalization","score":0.7789999842643738}],"concepts":[{"id":"https://openalex.org/C2778049539","wikidata":"https://www.wikidata.org/wiki/Q17002908","display_name":"Bayesian optimization","level":2,"score":0.6834283471107483},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.6699091196060181},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5458846092224121},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4110439121723175},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32830607891082764}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2403.19390","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.19390","pdf_url":"https://arxiv.org/pdf/2403.19390","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2403.19390","doi":"https://doi.org/10.48550/arxiv.2403.19390","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2403.19390","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2403.19390","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.19390","pdf_url":"https://arxiv.org/pdf/2403.19390","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G119245639","display_name":null,"funder_award_id":"62306087","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7899663717","display_name":null,"funder_award_id":"ZR2023QF154","funder_id":"https://openalex.org/F4320324174","funder_display_name":"Natural Science Foundation of Shandong Province"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320324174","display_name":"Natural Science Foundation of Shandong Province","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393336136.pdf","grobid_xml":"https://content.openalex.org/works/W4393336136.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W1555835443","https://openalex.org/W2014384147","https://openalex.org/W2099201756","https://openalex.org/W2736422900","https://openalex.org/W2738674657","https://openalex.org/W2764163036","https://openalex.org/W2794325560","https://openalex.org/W2873705236","https://openalex.org/W2883265831","https://openalex.org/W2912811302","https://openalex.org/W2948223045","https://openalex.org/W2951665052","https://openalex.org/W2973727699","https://openalex.org/W2989588035","https://openalex.org/W3083410900","https://openalex.org/W3114990944","https://openalex.org/W3129831491","https://openalex.org/W3194676777","https://openalex.org/W3214411122","https://openalex.org/W4221147360","https://openalex.org/W4221155125","https://openalex.org/W4226441840","https://openalex.org/W4312089761","https://openalex.org/W4322718191","https://openalex.org/W4327810158","https://openalex.org/W4362706828","https://openalex.org/W4386348101","https://openalex.org/W4386942538","https://openalex.org/W4387145092","https://openalex.org/W4387322933","https://openalex.org/W4387389872","https://openalex.org/W4388482013","https://openalex.org/W4390306040","https://openalex.org/W4390690464","https://openalex.org/W4391156874","https://openalex.org/W4391159098","https://openalex.org/W4406844855","https://openalex.org/W7096505009"],"related_works":["https://openalex.org/W4411957315","https://openalex.org/W7166869410","https://openalex.org/W4416400889","https://openalex.org/W4416445299","https://openalex.org/W7162506741","https://openalex.org/W4379919333","https://openalex.org/W4404307413","https://openalex.org/W4416004325","https://openalex.org/W7135018776","https://openalex.org/W7214203082"],"abstract_inverted_index":{"The":[0],"rapid":[1],"proliferation":[2],"of":[3],"large":[4],"language":[5],"models":[6],"(LLMs)":[7],"such":[8],"as":[9],"GPT-4":[10],"and":[11,30,53],"Gemini":[12],"underscores":[13],"the":[14,63,81],"intense":[15],"demand":[16],"for":[17,62],"resources":[18],"during":[19],"their":[20],"training":[21,51],"processes,":[22],"posing":[23],"significant":[24],"challenges":[25],"due":[26],"to":[27,83,90],"substantial":[28,92],"computational":[29],"environmental":[31],"costs.":[32],"To":[33],"alleviate":[34],"this":[35],"issue,":[36],"we":[37,73],"propose":[38],"checkpoint":[39],"merging":[40,65],"in":[41,56,118],"pretraining":[42],"LLM.":[43],"This":[44],"method":[45],"utilizes":[46],"LLM":[47],"checkpoints":[48],"with":[49],"shared":[50],"trajectories,":[52],"is":[54],"rooted":[55],"an":[57,87],"extensive":[58],"search":[59],"space":[60],"exploration":[61],"best":[64],"weight":[66],"via":[67],"Bayesian":[68],"optimization.":[69],"Through":[70],"various":[71],"experiments,":[72],"demonstrate":[74],"that:":[75],"(1)":[76],"Our":[77,98],"proposed":[78,99],"methodology":[79],"exhibits":[80],"capacity":[82],"augment":[84],"pretraining,":[85],"presenting":[86],"opportunity":[88],"akin":[89],"obtaining":[91],"benefits":[93],"at":[94],"minimal":[95],"cost;":[96],"(2)":[97],"methodology,":[100],"despite":[101],"requiring":[102],"a":[103,115],"given":[104],"held-out":[105],"dataset,":[106],"still":[107],"demonstrates":[108],"robust":[109],"generalization":[110],"capabilities":[111],"across":[112],"diverse":[113],"domains,":[114],"pivotal":[116],"aspect":[117],"pretraining.":[119]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4393336136","counts_by_year":[{"year":2026,"cited_by_count":15},{"year":2025,"cited_by_count":28},{"year":2024,"cited_by_count":3}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2024-03-30T00:00:00"}
