{"id":"https://openalex.org/W7224858822","doi":"https://doi.org/10.48550/arxiv.2610.40051","title":"Rethinking Pre-pretraining","display_name":"Rethinking Pre-pretraining","publication_year":2026,"publication_date":"2026-10-29","ids":{"openalex":"https://openalex.org/W7224858822","doi":"https://doi.org/10.48550/arxiv.2610.40051"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.40051","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.40051","pdf_url":"https://arxiv.org/pdf/2610.40051","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.40051","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5160224135","display_name":"Alexander Stern","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4401726847","display_name":"Cohere (Canada)","ror":"https://ror.org/050yr0479","country_code":"US","type":"company","lineage":["https://openalex.org/I4401726847"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stern, Alexander","raw_affiliation_strings":["Cohere"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cohere","institution_ids":["https://openalex.org/I4401726847"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079315903","display_name":"Tim Rockt\u00e4schel","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Rockt\u00e4schel, Tim","raw_affiliation_strings":["AI Centre , UCL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AI Centre , UCL","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003520377","display_name":"Acyr Locatelli","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Locatelli, Acyr","raw_affiliation_strings":["MIT"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5060010430","display_name":"Laura Ruis","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ruis, Laura","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8363999724388123,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8363999724388123,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10159999877214432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.029400000348687172,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computational-resource-allocation","display_name":"computing resource allocation","score":0.5929999947547913},{"id":"https://openalex.org/keywords/weight-initialization","display_name":"weight initialization","score":0.7200000286102295},{"id":"https://openalex.org/keywords/neural-cellular-automata","display_name":"neural cellular automata","score":0.9160000085830688},{"id":"https://openalex.org/keywords/dyck-languages","display_name":"Dyck languages","score":0.6980000138282776},{"id":"https://openalex.org/keywords/language-model-training","display_name":"language model training","score":0.7870000004768372}],"concepts":[{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.8988999724388123},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.8104000091552734},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6212999820709229},{"id":"https://openalex.org/C2778775528","wikidata":"https://www.wikidata.org/wiki/Q5135432","display_name":"Closing (real estate)","level":2,"score":0.5192000269889832},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5102999806404114},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5099999904632568},{"id":"https://openalex.org/C28034677","wikidata":"https://www.wikidata.org/wiki/Q17092530","display_name":"Interleaving","level":2,"score":0.4551999866962433},{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.44209998846054077},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.423799991607666},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4162999987602234},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4083000123500824},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.36090001463890076},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.34380000829696655},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.3427000045776367},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.3416000008583069},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33980000019073486},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.33649998903274536},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.2921000123023987},{"id":"https://openalex.org/C112505250","wikidata":"https://www.wikidata.org/wiki/Q787116","display_name":"Automaton","level":2,"score":0.28679999709129333},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.26579999923706055}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.40051","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.40051","pdf_url":"https://arxiv.org/pdf/2610.40051","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.40051","doi":"https://doi.org/10.48550/arxiv.2610.40051","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.40051","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.40051","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.40051","pdf_url":"https://arxiv.org/pdf/2610.40051","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7224858822.pdf","grobid_xml":"https://content.openalex.org/works/W7224858822.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7215096441","https://openalex.org/W7160951853","https://openalex.org/W7135075179","https://openalex.org/W7133229197","https://openalex.org/W4414818045","https://openalex.org/W7221651935","https://openalex.org/W7214930272","https://openalex.org/W7172481710","https://openalex.org/W7082653249","https://openalex.org/W4402684242"],"abstract_inverted_index":{"We":[0,78],"report":[1],"that":[2,80],"allocating":[3],"compute":[4,132],"to":[5,89],"synthetic":[6,64,189],"pre-pretraining":[7,190],"(PPT)":[8],"never":[9],"outperforms":[10],"pure":[11],"natural-language":[12,67,139,145,166,181],"training":[13,36,173,194],"in":[14,84,164,204],"our":[15],"compute-matched":[16],"experiments.":[17],"This":[18],"holds":[19],"across":[20],"four":[21],"PPT":[22,81,105,129,163],"curricula":[23],"(k-Dyck,":[24],"neural":[25],"cellular":[26],"automata":[27],"(NCA),":[28],"declarative":[29],"reference":[30],"corpora,":[31],"and":[32,34,40,196],"code)":[33],"three":[35],"regimes:":[37],"data-constrained,":[38],"data-unconstrained,":[39],"a":[41,59,100,135,192],"high":[42],"token-to-parameter-ratio":[43],"regime":[44],"(881M":[45],"parameters":[46],"trained":[47],"on":[48,66],"50B":[49],"tokens).":[50],"When":[51],"natural":[52],"language":[53,205],"is":[54],"scarce,":[55],"repeated":[56],"epochs":[57],"over":[58],"100M-token":[60],"corpus":[61],"beat":[62],"every":[63],"curriculum":[65],"validation":[68],"loss,":[69],"with":[70,148],"no":[71],"consistent":[72],"downstream":[73],"accuracy":[74],"advantage":[75],"from":[76,104],"PPT.":[77],"find":[79],"gains":[82,141],"reported":[83],"prior":[85],"work":[86],"are":[87,202],"sensitive":[88],"the":[90,107,118,123,138,169],"weight":[91],"initialization":[92,109],"scheme.":[93,110],"In":[94],"an":[95],"NCA":[96],"reproduction,":[97],"we":[98],"recover":[99],"5.4%":[101],"perplexity":[102],"improvement":[103],"under":[106],"original":[108],"An":[111],"alternative":[112],"scheme":[113],"improves":[114],"both":[115],"methods,":[116],"but":[117,151,176],"no-PPT":[119],"baseline":[120],"more,":[121],"closing":[122],"gap":[124],"between":[125],"them.":[126],"However,":[127],"when":[128],"receives":[130],"additional":[131],"rather":[133],"than":[134],"share":[136],"of":[137,188],"budget,":[140],"persist":[142],"at":[143],"small":[144],"budgets":[146],"even":[147],"improved":[149],"initialization,":[150],"disappear":[152],"by":[153],"1B":[154],"tokens.":[155],"Finally,":[156],"models":[157],"forget":[158],"what":[159],"they":[160],"learn":[161],"during":[162],"subsequent":[165],"training;":[167],"interleaving":[168],"same":[170],"data":[171,201],"throughout":[172],"prevents":[174],"forgetting,":[175],"does":[177],"not":[178],"significantly":[179],"improve":[180],"performance.":[182],"Our":[183],"results":[184],"suggest":[185],"limited":[186],"benefits":[187],"as":[191],"distinct":[193],"stage":[195],"motivate":[197],"rethinking":[198],"how":[199],"such":[200],"used":[203],"model":[206],"training.":[207]},"counts_by_year":[],"updated_date":"2026-11-05T12:25:35.769452","created_date":"2026-10-31T00:00:00"}