{"id":"https://openalex.org/W7214742307","doi":"https://doi.org/10.48550/arxiv.2609.33016","title":"Relative Generalization Invariance of LLM Pretraining","display_name":"Relative Generalization Invariance of LLM Pretraining","publication_year":2026,"publication_date":"2026-09-26","ids":{"openalex":"https://openalex.org/W7214742307","doi":"https://doi.org/10.48550/arxiv.2609.33016"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.33016","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.33016","pdf_url":"https://arxiv.org/pdf/2609.33016","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.33016","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5153449085","display_name":"Fengzhuo Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhang, Fengzhuo","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153429571","display_name":"Shuche Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Wang, Shuche","raw_affiliation_strings":["National University of Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153502000","display_name":"Shenggui Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Li, Shenggui","raw_affiliation_strings":["Nanyang Technological University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanyang Technological University","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120656279","display_name":"Tianyu Ruan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ruan, Tianyu","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153548939","display_name":"Jianliang He","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"He, Jianliang","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021751767","display_name":"Ivor Wai-Hung Tsang","orcid":"https://orcid.org/0000-0001-8095-4637","observed_orcids":["https://orcid.org/0000-0001-8095-4637","https://orcid.org/0000-0003-2211-8176"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tsang, Ivor","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031863731","display_name":"Tianyu Pang","orcid":"https://orcid.org/0000-0003-0639-6176","observed_orcids":["https://orcid.org/0000-0003-0639-6176"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pang, Tianyu","raw_affiliation_strings":["Sea AI Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sea AI Lab","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100566952","display_name":"Du Chao","orcid":"https://orcid.org/0000-0002-0453-1920","observed_orcids":["https://orcid.org/0000-0002-0453-1920"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du, Chao","raw_affiliation_strings":["Sea AI Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sea AI Lab","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153408322","display_name":"Tianwei Zhang","orcid":"https://orcid.org/0000-0002-1462-5402","observed_orcids":["https://orcid.org/0000-0002-1462-5402"]},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Zhang, Tianwei","raw_affiliation_strings":["Nanyang Technological University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanyang Technological University","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5153493811","display_name":"Zhuoran Yang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang, Zhuoran","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7397000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7397000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.1492999941110611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.053599998354911804,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-pretraining","display_name":"large language model pretraining","score":0.968999981880188},{"id":"https://openalex.org/keywords/neural-tangent-kernel","display_name":"neural tangent kernel","score":0.9229999780654907},{"id":"https://openalex.org/keywords/mean-field-regime","display_name":"mean-field regime","score":0.9459999799728394},{"id":"https://openalex.org/keywords/generalization","display_name":"generalization","score":0.5099999904632568}],"concepts":[{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.760200023651123},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.599399983882904},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48890000581741333},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.483599990606308},{"id":"https://openalex.org/C138187205","wikidata":"https://www.wikidata.org/wiki/Q131251","display_name":"Tangent","level":2,"score":0.4758000075817108},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.45210000872612},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.438400000333786},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.43130001425743103},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.4309000074863434},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3361999988555908},{"id":"https://openalex.org/C2776879701","wikidata":"https://www.wikidata.org/wiki/Q25048660","display_name":"Multiple kernel learning","level":4,"score":0.3244999945163727},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.31839999556541443},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3125},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3034999966621399},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.29190000891685486},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27709999680519104},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.2736000120639801},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.2651999890804291},{"id":"https://openalex.org/C2777413886","wikidata":"https://www.wikidata.org/wiki/Q3276013","display_name":"Fluency","level":2,"score":0.25589999556541443}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.33016","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.33016","pdf_url":"https://arxiv.org/pdf/2609.33016","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.33016","doi":"https://doi.org/10.48550/arxiv.2609.33016","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.33016","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.33016","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.33016","pdf_url":"https://arxiv.org/pdf/2609.33016","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214742307.pdf","grobid_xml":"https://content.openalex.org/works/W7214742307.grobid-xml"},"referenced_works_count":60,"referenced_works":["https://openalex.org/W6908809","https://openalex.org/W2785523195","https://openalex.org/W2803636134","https://openalex.org/W2894604724","https://openalex.org/W2896101215","https://openalex.org/W2899663614","https://openalex.org/W2964121744","https://openalex.org/W2994747787","https://openalex.org/W2996151541","https://openalex.org/W3001279689","https://openalex.org/W3128590981","https://openalex.org/W3196318247","https://openalex.org/W3211290194","https://openalex.org/W4293819916","https://openalex.org/W4378509498","https://openalex.org/W4389500833","https://openalex.org/W4391555935","https://openalex.org/W4392575839","https://openalex.org/W4393211960","https://openalex.org/W4393213783","https://openalex.org/W4394673581","https://openalex.org/W4394709684","https://openalex.org/W4399655876","https://openalex.org/W4400104096","https://openalex.org/W4400342789","https://openalex.org/W4400611281","https://openalex.org/W4403707245","https://openalex.org/W4403964581","https://openalex.org/W4407424967","https://openalex.org/W4407759068","https://openalex.org/W4414693706","https://openalex.org/W4414757763","https://openalex.org/W4414858766","https://openalex.org/W4414968665","https://openalex.org/W4414977079","https://openalex.org/W4414988195","https://openalex.org/W4415278947","https://openalex.org/W4415961075","https://openalex.org/W4416043542","https://openalex.org/W4416055854","https://openalex.org/W4416438542","https://openalex.org/W4416445221","https://openalex.org/W4416692582","https://openalex.org/W4416968619","https://openalex.org/W6891877040","https://openalex.org/W6929208038","https://openalex.org/W7128070483","https://openalex.org/W7128496784","https://openalex.org/W7129766187","https://openalex.org/W7130733643","https://openalex.org/W7131656391","https://openalex.org/W7133225605","https://openalex.org/W7133333687","https://openalex.org/W7133348289","https://openalex.org/W7134934673","https://openalex.org/W7135081841","https://openalex.org/W7139035261","https://openalex.org/W7140217466","https://openalex.org/W7154373369","https://openalex.org/W7196963086"],"related_works":["https://openalex.org/W7127359015","https://openalex.org/W4416372082","https://openalex.org/W4404405771","https://openalex.org/W7159546903","https://openalex.org/W4402670181","https://openalex.org/W7163514553","https://openalex.org/W7202178137","https://openalex.org/W4407759068","https://openalex.org/W4416526942","https://openalex.org/W7214726746"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Model":[2],"(LLM)":[3],"pretraining":[4,145],"performance":[5,29],"is":[6],"jointly":[7],"shaped":[8],"by":[9,44,114],"three":[10],"components":[11,27],"of":[12,56,76,151,157],"the":[13,16,54,57,97,115,149],"training":[14,21,98,158],"triplet:":[15],"optimizer,":[17],"model":[18],"architecture,":[19],"and":[20,78,123,153],"data":[22,99],"stream.":[23],"However,":[24],"how":[25],"these":[26,84],"influence":[28],"in":[30,91,129,143],"distinct":[31],"ways":[32],"remains":[33],"unclear.":[34],"We":[35,48,66,106],"take":[36],"a":[37,73,140],"first":[38],"step":[39],"toward":[40],"isolating":[41],"their":[42],"effects":[43,150],"studying":[45],"relative":[46,104],"generalization.":[47,105],"introduce":[49],"Relative":[50],"Generalization":[51],"Invariance":[52],"(RGI),":[53],"invariance":[55],"validation-loss":[58],"difference":[59],"between":[60],"any":[61],"two":[62],"tokens":[63],"across":[64,72],"models.":[65],"show":[67,108],"that":[68,83,109,125,146],"RGI":[69,110,138],"approximately":[70,88],"holds":[71],"wide":[74],"range":[75],"optimizers":[77,152],"moderate":[79],"architectural":[80],"variations,":[81],"suggesting":[82],"choices":[85],"induce":[86],"an":[87,130],"uniform":[89],"shift":[90],"token-wise":[92],"losses.":[93],"In":[94],"contrast,":[95],"changing":[96],"stream":[100],"can":[101,127],"substantially":[102],"alter":[103],"further":[107],"cannot":[111],"be":[112],"explained":[113],"neural":[116],"tangent":[117],"kernel":[118],"or":[119],"mean-field":[120],"regimes":[121],"alone":[122],"prove":[124],"it":[126],"emerge":[128],"overparameterized":[131],"quadratic":[132],"model.":[133],"Overall,":[134],"our":[135],"work":[136],"identifies":[137],"as":[139],"new":[141],"phenomenon":[142],"LLM":[144],"helps":[147],"distinguish":[148],"architectures":[154],"from":[155],"those":[156],"data.":[159]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-30T00:00:00"}