{"id":"https://openalex.org/W7155080150","doi":"https://doi.org/10.48550/arxiv.2604.16380","title":"Data Mixing for Large Language Models Pretraining: A Survey and Outlook","display_name":"Data Mixing for Large Language Models Pretraining: A Survey and Outlook","publication_year":2026,"publication_date":"2026-03-25","ids":{"openalex":"https://openalex.org/W7155080150","doi":"https://doi.org/10.48550/arxiv.2604.16380"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.16380","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.16380","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.16380","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134114080","display_name":"Zhuo Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Zhuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134118876","display_name":"Yuxuan Miao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Miao, Yuxuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134145903","display_name":"Supryadi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Supryadi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134137762","display_name":"Deyi Xiong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiong, Deyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.3034000098705292,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.3034000098705292,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2549999952316284,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.05869999900460243,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mixing","display_name":"Mixing (physics)","score":0.604200005531311},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.48730000853538513},{"id":"https://openalex.org/keywords/exploratory-data-analysis","display_name":"Exploratory data analysis","score":0.40939998626708984},{"id":"https://openalex.org/keywords/flexibility","display_name":"Flexibility (engineering)","score":0.3630000054836273},{"id":"https://openalex.org/keywords/a-priori-and-a-posteriori","display_name":"A priori and a posteriori","score":0.3628000020980835},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.34850001335144043},{"id":"https://openalex.org/keywords/synthetic-data","display_name":"Synthetic data","score":0.3294999897480011},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.3294000029563904}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7623999714851379},{"id":"https://openalex.org/C138777275","wikidata":"https://www.wikidata.org/wiki/Q6884054","display_name":"Mixing (physics)","level":2,"score":0.604200005531311},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.48730000853538513},{"id":"https://openalex.org/C120894424","wikidata":"https://www.wikidata.org/wiki/Q1322871","display_name":"Exploratory data analysis","level":2,"score":0.40939998626708984},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4090999960899353},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3977999985218048},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.3630000054836273},{"id":"https://openalex.org/C75553542","wikidata":"https://www.wikidata.org/wiki/Q178161","display_name":"A priori and a posteriori","level":2,"score":0.3628000020980835},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.34850001335144043},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.3294999897480011},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3294000029563904},{"id":"https://openalex.org/C55037315","wikidata":"https://www.wikidata.org/wiki/Q5421151","display_name":"Experimental data","level":2,"score":0.2992999851703644},{"id":"https://openalex.org/C138958017","wikidata":"https://www.wikidata.org/wiki/Q190087","display_name":"Data type","level":2,"score":0.28769999742507935},{"id":"https://openalex.org/C92446256","wikidata":"https://www.wikidata.org/wiki/Q3306762","display_name":"Data validation","level":2,"score":0.28439998626708984},{"id":"https://openalex.org/C198477413","wikidata":"https://www.wikidata.org/wiki/Q7647069","display_name":"Survey data collection","level":2,"score":0.2825999855995178},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.27889999747276306},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2784999907016754},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27399998903274536},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.2727999985218048},{"id":"https://openalex.org/C166955791","wikidata":"https://www.wikidata.org/wiki/Q629579","display_name":"Macro","level":2,"score":0.2667999863624573},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2655999958515167},{"id":"https://openalex.org/C5274069","wikidata":"https://www.wikidata.org/wiki/Q2285707","display_name":"Categorical variable","level":2,"score":0.25999999046325684},{"id":"https://openalex.org/C2781147490","wikidata":"https://www.wikidata.org/wiki/Q5156808","display_name":"Compositional data","level":2,"score":0.2547000050544739},{"id":"https://openalex.org/C2781395549","wikidata":"https://www.wikidata.org/wiki/Q4680762","display_name":"Adaptive sampling","level":3,"score":0.2540000081062317}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.16380","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.16380","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.16380","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.16380","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"rely":[4],"on":[5,7,20,99,183],"pretraining":[6,112],"massive":[8],"and":[9,23,29,71,103,114,150,160,171,175,202,211,213,220,236,249],"heterogeneous":[10],"corpora,":[11],"where":[12],"training":[13,21],"data":[14,30,35,37,60,84,92,108,197,238],"composition":[15],"has":[16,57],"a":[17,52,73,80,96,129,178],"decisive":[18],"impact":[19],"efficiency":[22],"downstream":[24],"generalization":[25],"under":[26],"realistic":[27],"compute":[28],"budget":[31],"constraints.":[32],"Unlike":[33],"sample-level":[34],"selection,":[36],"mixing":[38,61,85,109,144,155],"optimizes":[39],"domain-level":[40],"sampling":[41],"weights":[42],"to":[43,246],"allocate":[44],"limited":[45,194],"budgets":[46],"more":[47],"effectively.":[48],"In":[49],"recent":[50],"years,":[51],"growing":[53],"body":[54],"of":[55,83,107],"work":[56],"proposed":[58],"principled":[59],"methods":[62,119,135],"for":[63,86,252],"LLM":[64,87],"pretraining;":[65],"however,":[66],"the":[67,100,105,111,214],"literature":[68],"remains":[69],"fragmented":[70],"lacks":[72],"dedicated,":[74],"systematic":[75],"survey.":[76],"This":[77],"paper":[78],"provides":[79],"comprehensive":[81],"review":[82],"pretraining.":[88],"We":[89,126],"first":[90],"formalize":[91],"mixture":[93],"optimization":[94,199],"as":[95,205,207,240,242],"bilevel":[97],"problem":[98],"probability":[101],"simplex":[102],"clarify":[104],"role":[106],"in":[110,124,223],"pipeline,":[113],"briefly":[115],"explain":[116],"how":[117],"existing":[118,134],"make":[120],"this":[121,184],"formulation":[122],"tractable":[123],"practice.":[125],"then":[127],"introduce":[128],"fine-grained":[130],"taxonomy":[131],"that":[132,189],"organizes":[133],"along":[136],"two":[137],"dimensions:":[138],"static":[139],"versus":[140],"dynamic":[141,154],"mixing.":[142],"Static":[143],"is":[145,156],"further":[146],"categorized":[147],"into":[148,158],"rule-based":[149],"learning-based":[151,224],"methods,":[152,192],"while":[153],"grouped":[157],"adaptive":[159],"externally":[161],"guided":[162],"families.":[163],"For":[164],"each":[165],"class,":[166],"we":[167,186,227],"summarize":[168],"representative":[169],"approaches":[170],"analyze":[172],"their":[173],"strengths":[174],"limitations":[176],"from":[177],"performance-cost":[179],"trade-off":[180],"perspective.":[181],"Building":[182],"analysis,":[185],"highlight":[187],"challenges":[188],"cut":[190],"across":[191,196],"including":[193,232],"transferability":[195],"domains,":[198],"objectives,":[200],"models,":[201],"validation":[203],"sets,":[204],"well":[206,241],"unstandardized":[208],"evaluation":[209],"protocols":[210],"benchmarks,":[212],"inherent":[215],"tension":[216],"between":[217],"performance":[218],"gains":[219],"cost":[221],"control":[222],"methods.":[225],"Finally,":[226],"outline":[228],"several":[229],"exploratory":[230],"directions,":[231],"finer-grained":[233],"domain":[234],"partitioning":[235],"inverse":[237],"mixing,":[239],"pipeline-aware":[243],"designs,":[244],"aiming":[245],"provide":[247],"conceptual":[248],"methodological":[250],"insights":[251],"future":[253],"research.":[254]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-22T00:00:00"}