{"id":"https://openalex.org/W7163200956","doi":"https://doi.org/10.48550/arxiv.2606.01155","title":"When Data Is Scarce: Scaling Sparse Language Models with Repeated Training","display_name":"When Data Is Scarce: Scaling Sparse Language Models with Repeated Training","publication_year":2026,"publication_date":"2026-05-31","ids":{"openalex":"https://openalex.org/W7163200956","doi":"https://doi.org/10.48550/arxiv.2606.01155"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.01155","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.01155","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.01155","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111921306","display_name":"Boqian Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Boqian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137626082","display_name":"Qiao Xiao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Qiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007711261","display_name":"Patrik Okanovic","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Okanovic, Patrik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133535485","display_name":"Tomasz Sternal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sternal, Tomasz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137697629","display_name":"Maurice van Keulen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"van Keulen, Maurice","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137689348","display_name":"Mykola Pechenizkiy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pechenizkiy, Mykola","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027401676","display_name":"Elena Mocanu","orcid":"https://orcid.org/0000-0002-0856-579X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mocanu, Elena","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137699172","display_name":"Torsten Hoefler","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hoefler, Torsten","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5011045254","display_name":"Decebal Constantin Mocanu","orcid":"https://orcid.org/0000-0002-5636-7683"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mocanu, Decebal Constantin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11326","display_name":"Stock Market Forecasting Methods","score":0.24660000205039978,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11326","display_name":"Stock Market Forecasting Methods","score":0.24660000205039978,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.14869999885559082,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.048700001090765,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/extrapolation","display_name":"Extrapolation","score":0.7792999744415283},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.6272000074386597},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5529000163078308},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5327000021934509},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.48840001225471497},{"id":"https://openalex.org/keywords/sparse-matrix","display_name":"Sparse matrix","score":0.4715999960899353},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.46720001101493835}],"concepts":[{"id":"https://openalex.org/C132459708","wikidata":"https://www.wikidata.org/wiki/Q744069","display_name":"Extrapolation","level":2,"score":0.7792999744415283},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.697700023651123},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6272000074386597},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5529000163078308},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5327000021934509},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.48840001225471497},{"id":"https://openalex.org/C56372850","wikidata":"https://www.wikidata.org/wiki/Q1050404","display_name":"Sparse matrix","level":3,"score":0.4715999960899353},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.46720001101493835},{"id":"https://openalex.org/C124066611","wikidata":"https://www.wikidata.org/wiki/Q28684319","display_name":"Sparse approximation","level":2,"score":0.4571000039577484},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45669999718666077},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40880000591278076},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3849000036716461},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.34880000352859497},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.3181999921798706},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.31450000405311584},{"id":"https://openalex.org/C2988430800","wikidata":"https://www.wikidata.org/wiki/Q428971","display_name":"Scaling law","level":3,"score":0.3093000054359436},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3068999946117401},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.28630000352859497},{"id":"https://openalex.org/C21080849","wikidata":"https://www.wikidata.org/wiki/Q13611879","display_name":"Data point","level":2,"score":0.27480000257492065}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.01155","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.01155","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.01155","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.01155","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.42056339979171753,"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Scaling":[0],"laws":[1],"for":[2,165,170],"dense":[3],"LLMs":[4],"under":[5,174],"infinite":[6],"data":[7,17,54,107,117,121,156,175],"are":[8],"well":[9],"explored,":[10],"but":[11,167],"how":[12],"sparsity":[13,49,143,150,159],"interacts":[14],"with":[15,155],"limited":[16,31],"is":[18,144,151,160,179],"not.":[19],"In":[20],"this":[21],"work,":[22],"we":[23,70],"study":[24],"sparse":[25,123],"training":[26,62,124,133],"in":[27,45,88],"data-constrained":[28],"regimes":[29],"where":[30],"unique":[32,53,105],"tokens":[33,63],"require":[34],"multi-epoch":[35,132],"training.":[36],"Our":[37,177],"experiments":[38],"span":[39],"models":[40,77,97],"up":[41,50,56,64,78],"to":[42,51,57,65,79],"1.92B":[43],"parameters":[44],"the":[46],"fitting":[47],"set,":[48],"93.75%,":[52],"budgets":[55],"2.6B":[58],"tokens,":[59,106],"and":[60,109,116,153],"total":[61],"41.6B":[66],"over":[67],"16":[68],"epochs;":[69],"further":[71],"validate":[72],"extrapolation":[73],"on":[74],"held-out":[75],"dense-equivalent":[76],"7.68B":[80],"parameters.":[81],"We":[82,91],"find":[83],"that:":[84],"1.":[85],"Sparse":[86],"scaling":[87,94,172],"data-limited":[89],"settings:":[90],"introduce":[92],"a":[93,100,163,168],"law":[95],"that":[96],"loss":[98],"as":[99],"function":[101],"of":[102],"active":[103],"parameters,":[104],"repetition,":[108],"sparsity,":[110],"accurately":[111],"predicting":[112],"performance":[113],"across":[114],"compute":[115],"budgets.":[118],"2.":[119],"Delayed":[120],"saturation:":[122],"postpones":[125],"diminishing":[126],"returns":[127],"from":[128],"repeated":[129],"data,":[130,141],"making":[131],"more":[134],"effective.":[135],"3.":[136],"Resource":[137],"trade-offs:":[138],"With":[139],"fixed":[140],"loss-optimal":[142],"moderate":[145],"~":[146],"50%,":[147],"while":[148],"compute-optimal":[149],"higher":[152],"grows":[154],"scale.":[157],"Overall,":[158],"not":[161],"just":[162],"tool":[164],"efficiency,":[166],"mechanism":[169],"improving":[171],"trade-offs":[173],"scarcity.":[176],"code":[178],"available":[180],"at:":[181],"https://github.com/boqian333/sparse-dc-scaling.":[182]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-03T00:00:00"}