{"id":"https://openalex.org/W4385968027","doi":"https://doi.org/10.48550/arxiv.2308.00951","title":"From Sparse to Soft Mixtures of Experts","display_name":"From Sparse to Soft Mixtures of Experts","publication_year":2023,"publication_date":"2023-08-02","ids":{"openalex":"https://openalex.org/W4385968027","doi":"https://doi.org/10.48550/arxiv.2308.00951"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2308.00951","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.00951","pdf_url":"https://arxiv.org/pdf/2308.00951","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2308.00951","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001091279","display_name":"Joan Puigcerver","orcid":"https://orcid.org/0000-0002-1926-2233","observed_orcids":["https://orcid.org/0000-0002-1926-2233"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Puigcerver, Joan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101675603","display_name":"Carlos Riquelme","orcid":"https://orcid.org/0000-0003-4861-8355","observed_orcids":["https://orcid.org/0000-0003-4861-8355"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Riquelme, Carlos","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072796087","display_name":"Basil Mustafa","orcid":"https://orcid.org/0000-0001-7305-7890","observed_orcids":["https://orcid.org/0000-0001-7305-7890"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mustafa, Basil","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5068878643","display_name":"Neil Houlsby","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Houlsby, Neil","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":178,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9814000129699707,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9814000129699707,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.014800000004470348,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sparse-mixture-of-experts","display_name":"sparse mixture-of-experts","score":0.8809999823570251},{"id":"https://openalex.org/keywords/training-stability","display_name":"training stability","score":0.6119999885559082},{"id":"https://openalex.org/keywords/inference-cost-reduction","display_name":"inference cost reduction","score":0.5580000281333923},{"id":"https://openalex.org/keywords/visual-recognition","display_name":"visual recognition","score":0.5809999704360962},{"id":"https://openalex.org/keywords/vision-transformer","display_name":"vision transformer","score":0.6050000190734863},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.6510000228881836}],"concepts":[{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7649626135826111},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6946925520896912},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.47539934515953064},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4388819634914398},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4304935932159424},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4236915707588196},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.4184536933898926},{"id":"https://openalex.org/C46743427","wikidata":"https://www.wikidata.org/wiki/Q1341685","display_name":"Inference engine","level":3,"score":0.41660240292549133},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.16109055280685425},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12752965092658997},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.08603894710540771},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2308.00951","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.00951","pdf_url":"https://arxiv.org/pdf/2308.00951","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2308.00951","doi":"https://doi.org/10.48550/arxiv.2308.00951","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2308.00951","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2308.00951","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.00951","pdf_url":"https://arxiv.org/pdf/2308.00951","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385968027.pdf","grobid_xml":"https://content.openalex.org/works/W4385968027.grobid-xml"},"referenced_works_count":11,"referenced_works":["https://openalex.org/W2179423374","https://openalex.org/W2581624817","https://openalex.org/W3001279689","https://openalex.org/W3040573126","https://openalex.org/W3175343838","https://openalex.org/W3209532394","https://openalex.org/W4226161578","https://openalex.org/W4296406182","https://openalex.org/W4300980463","https://openalex.org/W7133225605","https://openalex.org/W7133227004"],"related_works":["https://openalex.org/W4402954127","https://openalex.org/W4402670023","https://openalex.org/W4398796940","https://openalex.org/W4386621109","https://openalex.org/W4417089127","https://openalex.org/W3170943566","https://openalex.org/W4289828103","https://openalex.org/W4304697563","https://openalex.org/W4225390052","https://openalex.org/W4392019778"],"abstract_inverted_index":{"Sparse":[0],"mixture":[1],"of":[2,25,36,60,74,93,110],"expert":[3],"architectures":[4],"(MoEs)":[5],"scale":[6,33],"model":[7,99],"capacity":[8,100],"without":[9],"significant":[10],"increases":[11],"in":[12,82,86,139],"training":[13,27],"or":[14,38],"inference":[15,105,155],"costs.":[16],"Despite":[17],"their":[18],"success,":[19],"MoEs":[20,122],"suffer":[21],"from":[22],"a":[23,48,91],"number":[24,35],"issues:":[26],"instability,":[28],"token":[29],"dropping,":[30],"inability":[31],"to":[32,78],"the":[34,58,94,108],"experts,":[37],"ineffective":[39],"finetuning.":[40],"In":[41,107],"this":[42],"work,":[43],"we":[44],"propose":[45],"Soft":[46,62,87,113,129,133],"MoE,":[47],"fully-differentiable":[49],"sparse":[50],"Transformer":[51],"that":[52],"addresses":[53],"these":[54],"challenges,":[55],"while":[56],"maintaining":[57],"benefits":[59],"MoEs.":[61],"MoE":[63,88,114,130,134,141],"performs":[64],"an":[65],"implicit":[66],"soft":[67],"assignment":[68],"by":[69],"passing":[70],"different":[71],"weighted":[72],"combinations":[73],"all":[75],"input":[76],"tokens":[77],"each":[79],"expert.":[80],"As":[81],"other":[83],"MoEs,":[84],"experts":[85,138],"only":[89,152],"process":[90],"subset":[92],"(combined)":[95],"tokens,":[96],"enabling":[97],"larger":[98],"(and":[101],"performance)":[102],"at":[103],"lower":[104],"cost.":[106],"context":[109],"visual":[111],"recognition,":[112],"greatly":[115],"outperforms":[116],"dense":[117],"Transformers":[118],"(ViTs)":[119],"and":[120,125,157],"popular":[121],"(Tokens":[123],"Choice":[124],"Experts":[126],"Choice).":[127],"Furthermore,":[128],"scales":[131],"well:":[132],"Huge/14":[135],"with":[136,151],"128":[137],"16":[140],"layers":[142],"has":[143],"over":[144],"40x":[145],"more":[146],"parameters":[147],"than":[148],"ViT":[149],"Huge/14,":[150],"2%":[153],"increased":[154],"time,":[156],"substantially":[158],"better":[159],"quality.":[160]},"counts_by_year":[{"year":2026,"cited_by_count":52},{"year":2025,"cited_by_count":68},{"year":2024,"cited_by_count":50},{"year":2023,"cited_by_count":8}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-08-19T00:00:00"}