{"id":"https://openalex.org/W7214988765","doi":"https://doi.org/10.48550/arxiv.2609.38011","title":"When do data mixtures improve scaling laws? Insights from high-dimensional regression","display_name":"When do data mixtures improve scaling laws? Insights from high-dimensional regression","publication_year":2026,"publication_date":"2026-09-29","ids":{"openalex":"https://openalex.org/W7214988765","doi":"https://doi.org/10.48550/arxiv.2609.38011"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.38011","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.38011","pdf_url":"https://arxiv.org/pdf/2609.38011","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.38011","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089835034","display_name":"Diyuan Wu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157556583","display_name":"Institute of Science and Technology Austria","ror":"https://ror.org/03gnh5541","country_code":"AT","type":"education","lineage":["https://openalex.org/I157556583"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Wu, Diyuan","raw_affiliation_strings":["Institute of Science and Technology Austria (ISTA) "],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Science and Technology Austria (ISTA) ","institution_ids":["https://openalex.org/I157556583"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128495209","display_name":"Lehan Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chen, Lehan","raw_affiliation_strings":["Department of Applied and Computational Mathematics , Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Applied and Computational Mathematics , Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063392474","display_name":"Theodor Misiakiewicz","orcid":"https://orcid.org/0000-0003-1470-7536","observed_orcids":["https://orcid.org/0000-0003-1470-7536"]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Misiakiewicz, Theodor","raw_affiliation_strings":["Department of Statistics and Data Science , Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Statistics and Data Science , Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061114394","display_name":"Marco Mondelli","orcid":"https://orcid.org/0000-0002-3242-7020","observed_orcids":["https://orcid.org/0000-0002-3242-7020","https://orcid.org/0009-0006-3003-1205"]},"institutions":[{"id":"https://openalex.org/I157556583","display_name":"Institute of Science and Technology Austria","ror":"https://ror.org/03gnh5541","country_code":"AT","type":"education","lineage":["https://openalex.org/I157556583"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Mondelli, Marco","raw_affiliation_strings":["Institute of Science and Technology Austria (ISTA) "],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Science and Technology Austria (ISTA) ","institution_ids":["https://openalex.org/I157556583"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.8711000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.8711000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.04360000044107437,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.019200000911951065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-mixing","display_name":"data mixing","score":0.7080000042915344},{"id":"https://openalex.org/keywords/scaling-laws","display_name":"scaling laws","score":0.7760000228881836},{"id":"https://openalex.org/keywords/high-dimensional-regression","display_name":"high-dimensional regression","score":0.6880000233650208},{"id":"https://openalex.org/keywords/auxiliary-information","display_name":"auxiliary information","score":0.6269999742507935},{"id":"https://openalex.org/keywords/minimax-risk","display_name":"minimax risk","score":0.625},{"id":"https://openalex.org/keywords/deterministic-equivalent","display_name":"deterministic equivalent","score":0.8809999823570251},{"id":"https://openalex.org/keywords/ridge-regression","display_name":"ridge regression","score":0.9869999885559082},{"id":"https://openalex.org/keywords/spectral-decay","display_name":"spectral decay","score":0.5669999718666077},{"id":"https://openalex.org/keywords/language-models","display_name":"language models","score":0.6660000085830688},{"id":"https://openalex.org/keywords/domain-generalization","display_name":"domain generalization","score":0.4230000078678131}],"concepts":[{"id":"https://openalex.org/C178650346","wikidata":"https://www.wikidata.org/wiki/Q201984","display_name":"Covariance","level":2,"score":0.6851000189781189},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6559000015258789},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5616999864578247},{"id":"https://openalex.org/C149728462","wikidata":"https://www.wikidata.org/wiki/Q751319","display_name":"Minimax","level":2,"score":0.5134999752044678},{"id":"https://openalex.org/C185142706","wikidata":"https://www.wikidata.org/wiki/Q1134404","display_name":"Covariance matrix","level":2,"score":0.46000000834465027},{"id":"https://openalex.org/C83546350","wikidata":"https://www.wikidata.org/wiki/Q1139051","display_name":"Regression","level":2,"score":0.41620001196861267},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.40059998631477356},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.3781999945640564},{"id":"https://openalex.org/C48921125","wikidata":"https://www.wikidata.org/wiki/Q10861030","display_name":"Linear regression","level":2,"score":0.37459999322891235},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.3578000068664551},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3319000005722046},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3163999915122986},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.31540000438690186},{"id":"https://openalex.org/C138777275","wikidata":"https://www.wikidata.org/wiki/Q6884054","display_name":"Mixing (physics)","level":2,"score":0.31360000371932983},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.30410000681877136},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.3003000020980835},{"id":"https://openalex.org/C70259352","wikidata":"https://www.wikidata.org/wiki/Q1847839","display_name":"Robust regression","level":3,"score":0.2935999929904938},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.28439998626708984},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.27709999680519104},{"id":"https://openalex.org/C180877172","wikidata":"https://www.wikidata.org/wiki/Q5401390","display_name":"Estimation of covariance matrices","level":3,"score":0.2718999981880188},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.25529998540878296},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.2547000050544739}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.38011","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.38011","pdf_url":"https://arxiv.org/pdf/2609.38011","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.38011","doi":"https://doi.org/10.48550/arxiv.2609.38011","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.38011","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.38011","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.38011","pdf_url":"https://arxiv.org/pdf/2609.38011","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320321181","display_name":"Austrian Science Fund","ror":"https://ror.org/013tf3c58"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214988765.pdf","grobid_xml":"https://content.openalex.org/works/W7214988765.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7164029780","https://openalex.org/W4415795766","https://openalex.org/W4417256404","https://openalex.org/W7164007801","https://openalex.org/W4407683676","https://openalex.org/W4415366986","https://openalex.org/W3034730172","https://openalex.org/W2788004794","https://openalex.org/W7222984212","https://openalex.org/W157611635"],"abstract_inverted_index":{"Modern":[0],"machine":[1],"learning":[2],"systems":[3],"are":[4],"trained":[5],"on":[6,27,195,216],"mixtures":[7,161,205],"of":[8,109,140,149,189],"data":[9,28,43,160,204],"from":[10],"different":[11,84],"domains,":[12],"and":[13,30,37,74,77,101,122,145,185],"choosing":[14],"the":[15,88,97,106,132,146,150,176,190,199],"right":[16],"mixture":[17],"can":[18],"substantially":[19],"improve":[20],"downstream":[21],"performance.":[22],"Despite":[23],"an":[24,92,123],"extensive":[25],"literature":[26],"mixing":[29],"reweighting,":[31],"existing":[32],"work":[33],"is":[34],"largely":[35],"empirical":[36],"it":[38],"remains":[39],"unclear":[40],"when":[41],"auxiliary":[42,124],"genuinely":[44],"improves":[45],"scaling":[46,136,166,177],"laws":[47,137,154],"rather":[48],"than":[49,168,214],"merely":[50],"providing":[51],"more":[52],"samples.":[53],"To":[54],"gain":[55],"insight":[56],"into":[57],"this":[58],"question,":[59],"we":[60],"study":[61],"a":[62,68,119,164,180,207],"high-dimensional":[63],"mixed-data":[64],"regression":[65,70,111],"model":[66],"with":[67,126],"shared":[69],"function,":[71],"heterogeneous":[72],"covariances":[73],"noise":[75],"levels,":[76],"dataset":[78,171],"sizes":[79,188],"that":[80],"may":[81],"grow":[82],"at":[83],"rates.":[85],"We":[86,115],"establish":[87],"minimax":[89],"risk":[90],"under":[91,112],"ellipsoidal":[93],"parameter":[94],"constraint":[95],"for":[96,105],"general":[98],"covariance":[99,129],"structure":[100],"derive":[102],"deterministic":[103],"equivalents":[104],"test":[107,212],"error":[108],"ridge":[110],"commutative":[113],"covariances.":[114],"then":[116],"specialize":[117],"to":[118],"target":[120,143],"domain":[121,125,218],"aligned":[127],"power-law":[128],"spectra,":[130],"where":[131],"theory":[133],"yields":[134,163],"explicit":[135],"in":[138,157,210],"terms":[139],"spectral":[141],"decay,":[142],"regularity,":[144],"relative":[147,186],"growth":[148],"two":[151],"datasets.":[152],"These":[153],"identify":[155],"regimes":[156],"which":[158],"combining":[159],"provably":[162],"faster":[165,208],"rate":[167],"using":[169],"either":[170,217],"alone.":[172,219],"In":[173],"particular,":[174],"improving":[175],"law":[178],"requires":[179],"specific":[181],"interplay":[182],"between":[183],"spectra":[184],"sample":[187],"domains.":[191],"Our":[192],"numerical":[193],"experiments":[194],"language":[196],"models":[197],"exhibit":[198],"same":[200],"qualitative":[201],"phenomenon:":[202],"appropriate":[203],"yield":[206],"decrease":[209],"target-domain":[211],"loss":[213],"training":[215]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-10-01T00:00:00"}