{"id":"https://openalex.org/W7164878874","doi":"https://doi.org/10.48550/arxiv.2606.16456","title":"SPRI: SVD-Partitioned Residual Initialization for Data-Constrained MoE Upcycling","display_name":"SPRI: SVD-Partitioned Residual Initialization for Data-Constrained MoE Upcycling","publication_year":2026,"publication_date":"2026-06-15","ids":{"openalex":"https://openalex.org/W7164878874","doi":"https://doi.org/10.48550/arxiv.2606.16456"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.16456","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.16456","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.16456","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065538178","display_name":"Weiqiao Shan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shan, Weiqiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128081939","display_name":"Ruixiang Mao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mao, Ruixiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032504077","display_name":"Yuang Li","orcid":"https://orcid.org/0000-0002-2244-0184"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yuang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138737487","display_name":"Yuhao Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yuhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138738929","display_name":"Yingfeng Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Yingfeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138729536","display_name":"Tong Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Tong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138718542","display_name":"Chen Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055085448","display_name":"Yucheng Qiao","orcid":"https://orcid.org/0000-0001-9167-1958"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiao, Yucheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103001186","display_name":"Chunxiang Jin","orcid":"https://orcid.org/0000-0001-6793-4914"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Chunxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138751706","display_name":"Yi Yuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Yi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138717526","display_name":"Jingdong Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jingdong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138705621","display_name":"Tong Xiao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Tong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138712874","display_name":"Jingbo Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Jingbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.2062000036239624,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.2062000036239624,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.15710000693798065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.13030000030994415,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/initialization","display_name":"Initialization","score":0.7954999804496765},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6407999992370605},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.613099992275238},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.48030000925064087},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.3894999921321869},{"id":"https://openalex.org/keywords/homogeneous","display_name":"Homogeneous","score":0.35040000081062317},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.3353999853134155},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.3059000074863434}],"concepts":[{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.7954999804496765},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7159000039100647},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6407999992370605},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.613099992275238},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5370000004768372},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.48030000925064087},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4408999979496002},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.3894999921321869},{"id":"https://openalex.org/C66882249","wikidata":"https://www.wikidata.org/wiki/Q169336","display_name":"Homogeneous","level":2,"score":0.35040000081062317},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3353999853134155},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.3059000074863434},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.30230000615119934},{"id":"https://openalex.org/C146249460","wikidata":"https://www.wikidata.org/wiki/Q2914991","display_name":"Fixation (population genetics)","level":3,"score":0.27959999442100525},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.27459999918937683},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2743000090122223},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2696000039577484},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.26899999380111694},{"id":"https://openalex.org/C2781235140","wikidata":"https://www.wikidata.org/wiki/Q275131","display_name":"Scratch","level":2,"score":0.2606000006198883},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.25949999690055847},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.2531000077724457}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.16456","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.16456","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.16456","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.16456","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.5410687923431396}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"models":[2,23,156],"enable":[3],"efficient":[4],"scaling,":[5],"but":[6],"training":[7,37,112],"them":[8],"from":[9,89],"scratch":[10],"remains":[11],"prohibitively":[12],"expensive.":[13],"MoE":[14,26,130,168],"upcycling":[15,30,62,131,169],"mitigates":[16],"this":[17,59,76],"cost":[18],"by":[19,157,171],"converting":[20],"pretrained":[21,56,65,90,104],"dense":[22,155],"into":[24],"sparse":[25],"models.":[27],"However,":[28],"existing":[29],"methods":[31],"typically":[32],"rely":[33],"on":[34,121],"large-scale":[35],"continued":[36],"and":[38,132,150,159,163,174],"often":[39],"perform":[40],"poorly":[41],"under":[42],"data-constrained":[43],"supervised":[44,127],"adaptation,":[45],"due":[46],"to":[47,55,114],"either":[48],"homogeneous":[49],"experts":[50],"or":[51],"overly":[52],"disruptive":[53],"perturbations":[54],"parameters.":[57],"In":[58],"setting,":[60],"effective":[61],"must":[63],"leverage":[64],"weight":[66],"structure":[67],"while":[68],"introducing":[69,98],"sufficient":[70],"diversity":[71,101],"among":[72],"routed":[73,96],"experts.":[74],"To":[75],"end,":[77],"we":[78],"propose":[79],"SVD-Partitioned":[80],"Residual":[81],"Initialization":[82],"(SPRI),":[83],"which":[84],"distributes":[85],"SVD-partitioned":[86],"residuals":[87],"derived":[88],"feed-forward":[91],"network":[92],"(FFN)":[93],"weights":[94],"across":[95,142],"experts,":[97],"controlled":[99],"expert":[100],"grounded":[102],"in":[103],"spectral":[105],"structure.":[106],"We":[107,118],"further":[108],"introduce":[109],"a":[110],"two-stage":[111],"strategy":[113],"improve":[115],"adaptation":[116],"stability.":[117],"evaluate":[119],"SPRI":[120,146],"multilingual":[122],"speech-to-text":[123],"translation,":[124],"where":[125],"limited":[126],"data":[128],"challenges":[129],"multiple":[133],"target":[134],"languages":[135],"provide":[136],"natural":[137],"routing":[138],"heterogeneity.":[139],"On":[140],"CoVoST2":[141],"15":[143],"En-to-XX":[144],"directions,":[145],"improves":[147],"average":[148],"BLEU":[149,173],"COMET":[151,176],"over":[152],"fully":[153],"fine-tuned":[154],"2.58":[158],"3.32":[160],"points,":[161],"respectively,":[162],"outperforms":[164],"the":[165],"prior":[166],"best":[167],"baseline":[170],"3.39":[172],"4.34":[175],"points.":[177]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-17T00:00:00"}