{"id":"https://openalex.org/W7140275045","doi":"https://doi.org/10.3724/2096-7004.di.2026.0055","title":"Data Mixing for Large Language Models Pretraining: A Survey and Outlook","display_name":"Data Mixing for Large Language Models Pretraining: A Survey and Outlook","publication_year":2026,"publication_date":"2026-03-01","ids":{"openalex":"https://openalex.org/W7140275045","doi":"https://doi.org/10.3724/2096-7004.di.2026.0055"},"language":"en","primary_location":{"id":"doi:10.3724/2096-7004.di.2026.0055","is_oa":true,"landing_page_url":"https://doi.org/10.3724/2096-7004.di.2026.0055","pdf_url":"https://www.sciengine.com/doi/pdf/F635D24C18904B0B8F5F8D0CDAF39A79","source":{"id":"https://openalex.org/S4210186383","display_name":"Data Intelligence","issn_l":"2096-7004","issn":["2096-7004","2641-435X"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Data Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://www.sciengine.com/doi/pdf/F635D24C18904B0B8F5F8D0CDAF39A79","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5130555782","display_name":"Zhuo Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhuo Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130562469","display_name":"Yuxuan Miao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuxuan Miao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5130577591","display_name":"Deyi Xiong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deyi Xiong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.34137781,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"8","issue":"1","first_page":"15","last_page":"55"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2615000009536743,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2615000009536743,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2574000060558319,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.060100000351667404,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mixing","display_name":"Mixing (physics)","score":0.5665000081062317},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.4903999865055084},{"id":"https://openalex.org/keywords/exploratory-data-analysis","display_name":"Exploratory data analysis","score":0.40459999442100525},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.3416000008583069},{"id":"https://openalex.org/keywords/survey-data-collection","display_name":"Survey data collection","score":0.32679998874664307},{"id":"https://openalex.org/keywords/compositional-data","display_name":"Compositional data","score":0.325300008058548},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.32109999656677246},{"id":"https://openalex.org/keywords/a-priori-and-a-posteriori","display_name":"A priori and a posteriori","score":0.3172999918460846}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7390000224113464},{"id":"https://openalex.org/C138777275","wikidata":"https://www.wikidata.org/wiki/Q6884054","display_name":"Mixing (physics)","level":2,"score":0.5665000081062317},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.4903999865055084},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40790000557899475},{"id":"https://openalex.org/C120894424","wikidata":"https://www.wikidata.org/wiki/Q1322871","display_name":"Exploratory data analysis","level":2,"score":0.40459999442100525},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4002000093460083},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.3416000008583069},{"id":"https://openalex.org/C198477413","wikidata":"https://www.wikidata.org/wiki/Q7647069","display_name":"Survey data collection","level":2,"score":0.32679998874664307},{"id":"https://openalex.org/C2781147490","wikidata":"https://www.wikidata.org/wiki/Q5156808","display_name":"Compositional data","level":2,"score":0.325300008058548},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.32109999656677246},{"id":"https://openalex.org/C75553542","wikidata":"https://www.wikidata.org/wiki/Q178161","display_name":"A priori and a posteriori","level":2,"score":0.3172999918460846},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.31700000166893005},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.31150001287460327},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.30320000648498535},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.3001999855041504},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2985999882221222},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.29589998722076416},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.27950000762939453},{"id":"https://openalex.org/C92446256","wikidata":"https://www.wikidata.org/wiki/Q3306762","display_name":"Data validation","level":2,"score":0.2782000005245209},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.27459999918937683},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.27399998903274536},{"id":"https://openalex.org/C43091099","wikidata":"https://www.wikidata.org/wiki/Q1067788","display_name":"Through-the-lens metering","level":3,"score":0.27129998803138733},{"id":"https://openalex.org/C55037315","wikidata":"https://www.wikidata.org/wiki/Q5421151","display_name":"Experimental data","level":2,"score":0.2624000012874603},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2558000087738037},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2500999867916107}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3724/2096-7004.di.2026.0055","is_oa":true,"landing_page_url":"https://doi.org/10.3724/2096-7004.di.2026.0055","pdf_url":"https://www.sciengine.com/doi/pdf/F635D24C18904B0B8F5F8D0CDAF39A79","source":{"id":"https://openalex.org/S4210186383","display_name":"Data Intelligence","issn_l":"2096-7004","issn":["2096-7004","2641-435X"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Data Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2604.16380","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2604.16380","pdf_url":"https://arxiv.org/pdf/2604.16380","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.3724/2096-7004.di.2026.0055","is_oa":true,"landing_page_url":"https://doi.org/10.3724/2096-7004.di.2026.0055","pdf_url":"https://www.sciengine.com/doi/pdf/F635D24C18904B0B8F5F8D0CDAF39A79","source":{"id":"https://openalex.org/S4210186383","display_name":"Data Intelligence","issn_l":"2096-7004","issn":["2096-7004","2641-435X"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Data Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1701952510","display_name":null,"funder_award_id":"2023YFE0116400","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7140275045.pdf","grobid_xml":"https://content.openalex.org/works/W7140275045.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"rely":[4],"on":[5,7,23,102,192],"pretraining":[6,115],"massive":[8],"and":[9,26,32,74,106,117,155,166,179,184,212,221,232,248,261],"highly":[10],"heterogeneous":[11],"corpora,":[12],"where":[13],"the":[14,70,103,108,114,226],"composition":[15],"of":[16,58,86,110,173],"training":[17,24],"data":[18,33,38,40,63,87,95,111,207,250],"has":[19,60],"a":[20,55,76,83,99,133,187],"decisive":[21],"impact":[22],"efficiency":[25],"downstream":[27],"generalization":[28],"under":[29],"realistic":[30],"compute":[31],"budget":[34],"constraints.":[35],"Unlike":[36],"samplelevel":[37],"selection,":[39],"mixing":[41,64,88,112,149,160],"optimizes":[42],"domain-level":[43],"sampling":[44],"weights":[45],"to":[46,258],"allocate":[47],"limited":[48,204],"budgets":[49],"more":[50],"effectively.":[51],"In":[52],"recent":[53],"years,":[54],"growing":[56],"body":[57],"work":[59],"proposed":[61],"principled":[62],"methods":[65,122,139],"for":[66,89,264],"LLMs":[67,90],"pretraining;":[68],"however,":[69],"literature":[71],"remains":[72],"fragmented":[73],"lacks":[75],"dedicated,":[77],"systematic":[78],"survey.":[79],"This":[80],"paper":[81],"provides":[82],"comprehensive":[84],"review":[85],"pretraining.":[91],"We":[92,130],"first":[93],"formalize":[94],"mixture":[96],"optimization":[97,209],"as":[98,215,217,223,225,252,254],"bilevel":[100,125],"problem":[101],"probability":[104],"simplex":[105],"clarify":[107],"role":[109],"in":[113,128,235],"pipeline,":[116],"briefly":[118],"explain":[119],"how":[120],"existing":[121,138],"make":[123],"this":[124,193],"formulation":[126],"tractable":[127],"practice.":[129],"then":[131],"introduce":[132],"fine-grained":[134],"taxonomy":[135],"that":[136,199],"organizes":[137],"along":[140],"two":[141],"main":[142],"dimensions:":[143],"static":[144],"versus":[145],"dynamic":[146,159],"mixing.":[147],"Static":[148],"is":[150,161],"further":[151,162],"categorized":[152],"into":[153,164],"rule-based":[154],"learning-based":[156,236],"methods,":[157,174,202],"while":[158],"grouped":[163],"adaptive":[165],"externally":[167],"guided":[168],"families.":[169],"For":[170],"each":[171],"class":[172],"we":[175,195,239],"summarize":[176],"representative":[177],"approaches":[178],"analyze":[180],"their":[181],"characteristics,":[182],"strengths,":[183],"limitations":[185],"from":[186],"performance-cost":[188],"trade-off":[189],"perspective.":[190],"Building":[191],"analysis,":[194],"highlight":[196],"key":[197],"challenges":[198],"cut":[200],"across":[201,206],"including":[203,244],"transferability":[205],"domains,":[208],"objectives,":[210],"models,":[211],"validation":[213],"sets,":[214],"well":[216,224,253],"unstandardized":[218],"evaluation":[219],"protocols":[220],"benchmarks,":[222],"inherent":[227],"tension":[228],"between":[229],"performance":[230],"gains":[231],"cost":[233],"control":[234],"methods.":[237],"Finally,":[238],"outline":[240],"several":[241],"exploratory":[242],"directions,":[243],"finer-grained":[245],"domain":[246],"partitioning":[247],"inverse":[249],"mixing,":[251],"pipeline-aware":[255],"designs,":[256],"aiming":[257],"provide":[259],"conceptual":[260],"methodological":[262],"insights":[263],"future":[265],"research.":[266]},"counts_by_year":[],"updated_date":"2026-07-24T06:01:01.537669","created_date":"2026-03-26T00:00:00"}