{"id":"https://openalex.org/W7156910228","doi":"https://doi.org/10.48550/arxiv.2604.23036","title":"Preserving Long-Tailed Expert Information in Mixture-of-Experts Tuning","display_name":"Preserving Long-Tailed Expert Information in Mixture-of-Experts Tuning","publication_year":2026,"publication_date":"2026-04-24","ids":{"openalex":"https://openalex.org/W7156910228","doi":"https://doi.org/10.48550/arxiv.2604.23036"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.23036","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.23036","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.23036","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134810440","display_name":"Haoze He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Haoze","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066294648","display_name":"\u4e01\u674f\u6c85","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Xingyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134816324","display_name":"Xuan Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Xuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133625368","display_name":"Xinkai Zou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zou, Xinkai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134749913","display_name":"Alex Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Alex","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134789514","display_name":"Yibo Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yibo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111213544","display_name":"Juncheng Billy Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Juncheng Billy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134817032","display_name":"Heather Miller","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Miller, Heather","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10714","display_name":"Software-Defined Networks and 5G","score":0.1265999972820282,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10714","display_name":"Software-Defined Networks and 5G","score":0.1265999972820282,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.11410000175237656,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.10189999639987946,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/downstream","display_name":"Downstream (manufacturing)","score":0.6111999750137329},{"id":"https://openalex.org/keywords/encode","display_name":"ENCODE","score":0.5633999705314636},{"id":"https://openalex.org/keywords/router","display_name":"Router","score":0.49230000376701355},{"id":"https://openalex.org/keywords/information-gain","display_name":"Information gain","score":0.36559998989105225},{"id":"https://openalex.org/keywords/expert-system","display_name":"Expert system","score":0.35249999165534973}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6431999802589417},{"id":"https://openalex.org/C2776207758","wikidata":"https://www.wikidata.org/wiki/Q5303302","display_name":"Downstream (manufacturing)","level":2,"score":0.6111999750137329},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.5633999705314636},{"id":"https://openalex.org/C2775896111","wikidata":"https://www.wikidata.org/wiki/Q642560","display_name":"Router","level":2,"score":0.49230000376701355},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42719998955726624},{"id":"https://openalex.org/C2983203078","wikidata":"https://www.wikidata.org/wiki/Q255166","display_name":"Information gain","level":2,"score":0.36559998989105225},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.35249999165534973},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3301999866962433},{"id":"https://openalex.org/C191172861","wikidata":"https://www.wikidata.org/wiki/Q7899321","display_name":"Upstream (networking)","level":2,"score":0.274399995803833},{"id":"https://openalex.org/C2776014549","wikidata":"https://www.wikidata.org/wiki/Q3050847","display_name":"Consolidation (business)","level":2,"score":0.26089999079704285},{"id":"https://openalex.org/C52622258","wikidata":"https://www.wikidata.org/wiki/Q131222","display_name":"Information theory","level":2,"score":0.25519999861717224}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.23036","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.23036","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.23036","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.23036","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Despite":[0],"MoE":[1,11,96,173],"models":[2,174],"leading":[3],"many":[4],"benchmarks,":[5],"supervised":[6],"fine-tuning":[7],"(SFT)":[8],"for":[9,86],"the":[10],"architectures":[12],"remains":[13],"difficult":[14],"because":[15],"its":[16],"router":[17,28],"layers":[18],"are":[19,60],"fragile.":[20],"Methods":[21],"such":[22,183],"as":[23,184],"DenseMixer":[24,185],"and":[25,53,142,186,197],"ESFT":[26],"mitigate":[27],"collapse":[29],"with":[30,103],"dense":[31],"mixing":[32],"or":[33],"auxiliary":[34],"load-balancing":[35],"losses,":[36],"but":[37],"these":[38],"introduce":[39],"noisy":[40],"gradients":[41],"that":[42,55,77,99,138,147,159,176],"often":[43],"degrade":[44],"performance.":[45],"In":[46],"preliminary":[47],"experiments,":[48],"we":[49,92],"systematically":[50],"pruned":[51],"experts":[52,59,68,81,120,127,132],"observed":[54],"while":[56,124],"certain":[57],"super":[58],"activated":[61,80,154],"far":[62],"more":[63],"frequently,":[64],"discarding":[65],"less":[66],"used":[67],"still":[69],"leads":[70],"to":[71,121],"notable":[72],"performance":[73],"degradation.":[74],"This":[75],"suggests":[76],"even":[78],"rarely":[79],"encode":[82],"non-trivial":[83],"knowledge":[84],"useful":[85],"downstream":[87],"tasks.":[88],"Motivated":[89],"by":[90],"this,":[91],"propose":[93],"an":[94],"auxiliary-loss-free":[95],"SFT":[97,181],"framework":[98],"combines":[100],"bias-driven":[101],"sparsification":[102],"always-active":[104],"gated":[105],"condenser":[106,131],"experts.":[107,155],"Rather":[108],"than":[109],"enforcing":[110],"balanced":[111],"activation":[112],"across":[113,152],"all":[114],"experts,":[115],"our":[116,177],"method":[117],"encourages":[118],"task-relevant":[119],"remain":[122,150],"active":[123],"pushing":[125],"long-tailed":[126,164],"toward":[128],"inactivity.":[129],"The":[130],"provide":[133],"a":[134],"persistent,":[135],"learnable":[136],"pathway":[137],"alleviates":[139],"gradient":[140],"starvation":[141],"facilitates":[143],"consolidation":[144],"of":[145,191],"information":[146,166],"would":[148],"otherwise":[149],"fragmented":[151],"sparsely":[153],"Analysis":[156],"further":[157],"suggest":[158],"this":[160],"design":[161],"better":[162],"preserves":[163],"expert":[165],"under":[167],"sparse":[168],"routing.":[169],"Experiments":[170],"on":[171,193],"large-scale":[172],"demonstrate":[175],"approach":[178],"outperforms":[179],"state-of-the-art":[180],"baselines":[182],"ESFT,":[187],"achieving":[188],"average":[189],"gain":[190],"2.5%+":[192],"both":[194],"mathematical":[195],"reasoning":[196],"commonsenseQA":[198],"benchmarks.":[199]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-29T00:00:00"}