{"id":"https://openalex.org/W7154618995","doi":"https://doi.org/10.48550/arxiv.2604.13977","title":"How Can We Synthesize High-Quality Pretraining Data? A Systematic Study of Prompt Design, Generator Model, and Source Data","display_name":"How Can We Synthesize High-Quality Pretraining Data? A Systematic Study of Prompt Design, Generator Model, and Source Data","publication_year":2026,"publication_date":"2026-04-15","ids":{"openalex":"https://openalex.org/W7154618995","doi":"https://doi.org/10.48550/arxiv.2604.13977"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.13977","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.13977","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.13977","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133817747","display_name":"Joel Niklaus","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Niklaus, Joel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133795107","display_name":"Atsuki Yamaguchi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yamaguchi, Atsuki","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133752344","display_name":"Michal \u0160tef\u00e1nik","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"\u0160tef\u00e1nik, Michal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5105891900","display_name":"Guilherme Penedo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Penedo, Guilherme","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133759688","display_name":"Hynek Kydl\u00ed\u010dek","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kydl\u00ed\u010dek, Hynek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098954171","display_name":"Elie Bakouch","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bakouch, Elie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063084634","display_name":"Lewis Tunstall","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tunstall, Lewis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133758133","display_name":"Edward Emanuel Beeching","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Beeching, Edward Emanuel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120281387","display_name":"Thibaud Frere","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Frere, Thibaud","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045077843","display_name":"Colin Raffel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Raffel, Colin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133793864","display_name":"Leandro von Werra","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"von Werra, Leandro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133821677","display_name":"Thomas Wolf","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wolf, Thomas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.21299999952316284,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.21299999952316284,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.14429999887943268,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.07609999924898148,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/generator","display_name":"Generator (circuit theory)","score":0.8044999837875366},{"id":"https://openalex.org/keywords/synthetic-data","display_name":"Synthetic data","score":0.6258999705314636},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.5871000289916992},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.5156999826431274},{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"Natural language generation","score":0.4805000126361847},{"id":"https://openalex.org/keywords/web-application","display_name":"Web application","score":0.38519999384880066},{"id":"https://openalex.org/keywords/data-source","display_name":"Data source","score":0.382099986076355},{"id":"https://openalex.org/keywords/mixing","display_name":"Mixing (physics)","score":0.3707999885082245}],"concepts":[{"id":"https://openalex.org/C2780992000","wikidata":"https://www.wikidata.org/wiki/Q17016113","display_name":"Generator (circuit theory)","level":3,"score":0.8044999837875366},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7440999746322632},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.6258999705314636},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.5871000289916992},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.5156999826431274},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.4805000126361847},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41589999198913574},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.396699994802475},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.390500009059906},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.38519999384880066},{"id":"https://openalex.org/C2983685735","wikidata":"https://www.wikidata.org/wiki/Q5227355","display_name":"Data source","level":2,"score":0.382099986076355},{"id":"https://openalex.org/C138777275","wikidata":"https://www.wikidata.org/wiki/Q6884054","display_name":"Mixing (physics)","level":2,"score":0.3707999885082245},{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.35120001435279846},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3450999855995178},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.29989999532699585},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.27950000762939453},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2736000120639801},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.27309998869895935},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.25589999556541443},{"id":"https://openalex.org/C2777466982","wikidata":"https://www.wikidata.org/wiki/Q5227287","display_name":"Data extraction","level":3,"score":0.25279998779296875},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.25200000405311584}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.13977","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.13977","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.13977","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.13977","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.5969524383544922,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Synthetic":[0],"data":[1,99,129],"is":[2],"a":[3,113],"standard":[4],"component":[5],"in":[6,41],"training":[7],"large":[8],"language":[9],"models,":[10],"yet":[11],"systematic":[12],"comparisons":[13],"across":[14],"design":[15],"dimensions,":[16],"including":[17],"rephrasing":[18,42],"strategy,":[19],"generator":[20,80],"model,":[21],"and":[22,62,70,146],"source":[23],"data,":[24],"remain":[25],"absent.":[26],"We":[27,121,140],"conduct":[28],"extensive":[29],"controlled":[30],"experiments,":[31],"generating":[32],"over":[33],"one":[34],"trillion":[35],"tokens,":[36],"to":[37,137,150],"identify":[38],"critical":[39],"factors":[40],"web":[43,68,119],"text":[44],"into":[45],"synthetic":[46,72,128],"pretraining":[47],"data.":[48],"Our":[49,89],"results":[50],"reveal":[51],"that":[52,93,123],"structured":[53],"output":[54],"formats,":[55],"such":[56],"as":[57],"tables,":[58],"math":[59],"problems,":[60],"FAQs,":[61],"tutorials,":[63],"consistently":[64],"outperform":[65],"both":[66],"curated":[67],"baselines":[69,130],"prior":[71],"methods.":[73],"Notably,":[74],"increasing":[75],"the":[76,79,94,97,142,147,151],"size":[77],"of":[78,96,117],"model":[81],"beyond":[82],"1B":[83],"parameters":[84],"provides":[85],"no":[86],"additional":[87],"benefit.":[88],"analysis":[90],"also":[91],"demonstrates":[92],"selection":[95],"original":[98],"used":[100],"for":[101],"mixing":[102],"substantially":[103],"influences":[104],"performance.":[105],"By":[106],"applying":[107],"our":[108],"findings,":[109],"we":[110],"develop":[111],"\\textbf{\\textsc{FinePhrase}},":[112],"486-billion-token":[114],"open":[115],"dataset":[116],"rephrased":[118],"text.":[120],"show":[122],"\\textsc{FinePhrase}":[124],"outperforms":[125],"all":[126,144],"existing":[127],"while":[131],"reducing":[132],"generation":[133,148],"costs":[134],"by":[135],"up":[136],"30":[138],"times.":[139],"provide":[141],"dataset,":[143],"prompts,":[145],"framework":[149],"research":[152],"community.":[153]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-17T00:00:00"}