{"id":"https://openalex.org/W7160951853","doi":"https://doi.org/10.48550/arxiv.2605.10129","title":"Synthetic Pre-Pre-Training Improves Language Model Robustness to Noisy Pre-Training Data","display_name":"Synthetic Pre-Pre-Training Improves Language Model Robustness to Noisy Pre-Training Data","publication_year":2026,"publication_date":"2026-05-11","ids":{"openalex":"https://openalex.org/W7160951853","doi":"https://doi.org/10.48550/arxiv.2605.10129"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.10129","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.10129","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.10129","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135982031","display_name":"Xu Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135925410","display_name":"Runyu Peng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Runyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025411088","display_name":"Jian Tong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tong, Jian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135912688","display_name":"Yunhua Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Yunhua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135973950","display_name":"Haijun Lv","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lv, Haijun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135949534","display_name":"Zhihui Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Zhihui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135980124","display_name":"Qipeng Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Qipeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3515999913215637,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3515999913215637,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10320000350475311,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.07590000331401825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7946000099182129},{"id":"https://openalex.org/keywords/noisy-data","display_name":"Noisy data","score":0.6675000190734863},{"id":"https://openalex.org/keywords/synthetic-data","display_name":"Synthetic data","score":0.6262999773025513},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.6212000250816345},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4754999876022339},{"id":"https://openalex.org/keywords/artificial-noise","display_name":"Artificial noise","score":0.44839999079704285},{"id":"https://openalex.org/keywords/noise-measurement","display_name":"Noise measurement","score":0.41119998693466187},{"id":"https://openalex.org/keywords/data-modeling","display_name":"Data modeling","score":0.3716000020503998}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7946000099182129},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7235999703407288},{"id":"https://openalex.org/C2781170535","wikidata":"https://www.wikidata.org/wiki/Q30587856","display_name":"Noisy data","level":2,"score":0.6675000190734863},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.6262999773025513},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.6212000250816345},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5020999908447266},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4754999876022339},{"id":"https://openalex.org/C2780909371","wikidata":"https://www.wikidata.org/wiki/Q4801092","display_name":"Artificial noise","level":4,"score":0.44839999079704285},{"id":"https://openalex.org/C29265498","wikidata":"https://www.wikidata.org/wiki/Q7047719","display_name":"Noise measurement","level":3,"score":0.41119998693466187},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.40950000286102295},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.3716000020503998},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3714999854564667},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.3675999939441681},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3646000027656555},{"id":"https://openalex.org/C100675267","wikidata":"https://www.wikidata.org/wiki/Q1371624","display_name":"Background noise","level":2,"score":0.3492000102996826},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.31029999256134033},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3100000023841858},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2989000082015991},{"id":"https://openalex.org/C67226441","wikidata":"https://www.wikidata.org/wiki/Q1665389","display_name":"Robust statistics","level":3,"score":0.2888999879360199},{"id":"https://openalex.org/C163294075","wikidata":"https://www.wikidata.org/wiki/Q581861","display_name":"Noise reduction","level":2,"score":0.28700000047683716},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.27869999408721924},{"id":"https://openalex.org/C193519340","wikidata":"https://www.wikidata.org/wiki/Q891179","display_name":"Data loss","level":2,"score":0.27570000290870667},{"id":"https://openalex.org/C2984485829","wikidata":"https://www.wikidata.org/wiki/Q100766421","display_name":"Noise level","level":3,"score":0.2727000117301941},{"id":"https://openalex.org/C2988416141","wikidata":"https://www.wikidata.org/wiki/Q6031139","display_name":"Information loss","level":2,"score":0.27079999446868896},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.259799987077713},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2590000033378601}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.10129","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.10129","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.10129","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.10129","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.8224916458129883,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,135],"(LLMs)":[3],"rely":[4],"on":[5,51],"web-scale":[6],"corpora":[7,36],"for":[8],"pre-training.":[9],"The":[10],"noise":[11,77,86,119,151],"inherent":[12],"in":[13,39],"these":[14],"datasets":[15],"tends":[16],"to":[17,76,111,130],"obscure":[18],"meaningful":[19],"patterns":[20],"and":[21,153],"ultimately":[22],"degrade":[23],"model":[24],"performance.":[25],"Data":[26],"curation":[27],"mitigates":[28],"but":[29],"cannot":[30],"eliminate":[31],"such":[32],"noise,":[33],"so":[34],"pre-training":[35,64],"remain":[37],"noisy":[38,60,131,143],"practice.":[40],"We":[41],"therefore":[42],"study":[43],"whether":[44],"a":[45,89,92],"lightweight":[46],"pre-pre-training":[47],"(PPT)":[48],"stage":[49,95],"based":[50],"synthetic":[52,93,148],"data":[53,61],"with":[54,80,96],"learnable":[55],"temporal":[56],"structure":[57],"helps":[58],"resist":[59],"during":[62,78,142],"the":[63,101,106,155],"(PT)":[65],"stage.":[66],"Across":[67],"various":[68],"corruption":[69],"settings,":[70],"our":[71],"method":[72],"consistently":[73],"improves":[74],"robustness":[75],"PT,":[79],"larger":[81],"relative":[82],"gains":[83],"at":[84,162],"higher":[85],"levels.":[87,120],"For":[88],"1B-parameter":[90],"model,":[91],"PPT":[94,124,149],"only":[97],"65M":[98],"tokens":[99,116,141],"achieves":[100],"same":[102],"final":[103],"loss":[104],"as":[105],"baseline":[107],"while":[108],"using":[109],"up":[110],"49\\%":[112],"fewer":[113],"natural-text":[114],"PT":[115],"across":[117],"different":[118],"Mechanistic":[121],"analyses":[122],"suggest":[123],"does":[125],"not":[126],"immediately":[127],"suppress":[128],"attention":[129,138],"tokens.":[132],"Rather,":[133],"PPT-initialized":[134],"gradually":[136],"downweight":[137],"between":[139],"corrupted":[140],"PT.":[144],"This":[145],"indicates":[146],"that":[147],"inhibits":[150],"self-modeling":[152],"shapes":[154],"subsequent":[156],"optimization":[157],"trajectory.":[158],"Code":[159],"is":[160],"available":[161],"https://github.com/guox18/formal-language-prepretraining.":[163]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-13T00:00:00"}