{"id":"https://openalex.org/W4390437677","doi":"https://doi.org/10.48550/arxiv.2312.16903","title":"Spike No More: Stabilizing the Pre-training of Large Language Models","display_name":"Spike No More: Stabilizing the Pre-training of Large Language Models","publication_year":2023,"publication_date":"2023-12-28","ids":{"openalex":"https://openalex.org/W4390437677","doi":"https://doi.org/10.48550/arxiv.2312.16903"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2312.16903","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2312.16903","pdf_url":"https://arxiv.org/pdf/2312.16903","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2312.16903","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075222372","display_name":"Sho Takase","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201537933","display_name":"Tohoku University","ror":"https://ror.org/01dq60k83","country_code":"JP","type":"education","lineage":["https://openalex.org/I201537933"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Takase, Sho","raw_affiliation_strings":["Tohoku University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tohoku University","institution_ids":["https://openalex.org/I201537933"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066002127","display_name":"Shun Kiyono","orcid":"https://orcid.org/0000-0002-5063-9297","observed_orcids":["https://orcid.org/0000-0002-5063-9297"]},"institutions":[{"id":"https://openalex.org/I201537933","display_name":"Tohoku University","ror":"https://ror.org/01dq60k83","country_code":"JP","type":"education","lineage":["https://openalex.org/I201537933"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Kiyono, Shun","raw_affiliation_strings":["Tohoku University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tohoku University","institution_ids":["https://openalex.org/I201537933"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112876488","display_name":"Sosuke Kobayashi","orcid":"https://orcid.org/0009-0004-2087-6162","observed_orcids":["https://orcid.org/0009-0004-2087-6162"]},"institutions":[{"id":"https://openalex.org/I201537933","display_name":"Tohoku University","ror":"https://ror.org/01dq60k83","country_code":"JP","type":"education","lineage":["https://openalex.org/I201537933"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Kobayashi, Sosuke","raw_affiliation_strings":["Tohoku University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tohoku University","institution_ids":["https://openalex.org/I201537933"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5002182453","display_name":"Jun Suzuki","orcid":"https://orcid.org/0000-0003-2108-1340","observed_orcids":["https://orcid.org/0000-0003-2108-1340"]},"institutions":[{"id":"https://openalex.org/I201537933","display_name":"Tohoku University","ror":"https://ror.org/01dq60k83","country_code":"JP","type":"education","lineage":["https://openalex.org/I201537933"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Suzuki, Jun","raw_affiliation_strings":["Tohoku University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tohoku University","institution_ids":["https://openalex.org/I201537933"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I201537933"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":52,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8237000107765198,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8237000107765198,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.07370000332593918,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.02239999920129776,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-pretraining","display_name":"large language model pretraining","score":0.8709999918937683},{"id":"https://openalex.org/keywords/gradient-norm","display_name":"gradient norm","score":0.8309999704360962},{"id":"https://openalex.org/keywords/skip-connections","display_name":"skip connections","score":0.4580000042915344},{"id":"https://openalex.org/keywords/training-stability","display_name":"training stability","score":0.6629999876022339}],"concepts":[{"id":"https://openalex.org/C2781390188","wikidata":"https://www.wikidata.org/wiki/Q25203449","display_name":"Spike (software development)","level":2,"score":0.8818821907043457},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.8794422149658203},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8119235634803772},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.7432844638824463},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.720954418182373},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.6294959187507629},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4454779624938965},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.43124920129776},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3962153196334839},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.390936940908432},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.07557570934295654},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0728236734867096},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2312.16903","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2312.16903","pdf_url":"https://arxiv.org/pdf/2312.16903","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2312.16903","doi":"https://doi.org/10.48550/arxiv.2312.16903","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2312.16903","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2312.16903","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2312.16903","pdf_url":"https://arxiv.org/pdf/2312.16903","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4390437677.pdf","grobid_xml":"https://content.openalex.org/works/W4390437677.grobid-xml"},"referenced_works_count":2,"referenced_works":["https://openalex.org/W2794325560","https://openalex.org/W4388090547"],"related_works":["https://openalex.org/W4407690036","https://openalex.org/W7162569926","https://openalex.org/W6929284762","https://openalex.org/W7127359426","https://openalex.org/W4404782271","https://openalex.org/W7221561542","https://openalex.org/W4410282727","https://openalex.org/W4415185621","https://openalex.org/W4404782398","https://openalex.org/W4414847492"],"abstract_inverted_index":{"Loss":[0],"spikes":[1,11,115],"often":[2],"occur":[3],"during":[4,116],"pre-training":[5,26,84],"of":[6,15,51,67,71],"large":[7,16,92],"language":[8,17],"models.":[9],"The":[10],"degrade":[12],"the":[13,22,25,39,42,48,52,60,68,72,76,83,110],"performance":[14],"models":[18],"and":[19,91],"sometimes":[20],"ruin":[21],"pre-training.":[23,117],"Since":[24],"needs":[27],"a":[28],"vast":[29],"computational":[30],"budget,":[31],"we":[32,55],"should":[33],"avoid":[34],"such":[35],"spikes.":[36],"Based":[37],"on":[38],"assumption":[40],"that":[41,81,107],"loss":[43,114],"spike":[44],"is":[45],"caused":[46],"by":[47],"sudden":[49],"growth":[50],"gradient":[53,61],"norm,":[54],"explore":[56],"factors":[57],"to":[58,98],"keep":[59],"norm":[62],"small":[63,89],"through":[64],"an":[65],"analysis":[66],"spectral":[69],"norms":[70],"Jacobian":[73],"matrices":[74],"for":[75],"sub-layers.":[77],"Our":[78],"findings":[79],"suggest":[80],"stabilizing":[82],"process":[85],"requires":[86],"two":[87],"conditions:":[88],"sub-layers":[90],"shortcut.":[93],"We":[94],"conduct":[95],"various":[96],"experiments":[97],"empirically":[99],"verify":[100],"our":[101],"theoretical":[102],"analyses.":[103],"Experimental":[104],"results":[105],"demonstrate":[106],"methods":[108],"satisfying":[109],"conditions":[111],"effectively":[112],"prevent":[113]},"counts_by_year":[{"year":2026,"cited_by_count":20},{"year":2025,"cited_by_count":21},{"year":2024,"cited_by_count":11}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-12-30T00:00:00"}