{"id":"https://openalex.org/W7163136233","doi":"https://doi.org/10.48550/arxiv.2606.01509","title":"ProbMoE: Differentiable Probabilistic Routing for Mixture-of-Experts","display_name":"ProbMoE: Differentiable Probabilistic Routing for Mixture-of-Experts","publication_year":2026,"publication_date":"2026-06-01","ids":{"openalex":"https://openalex.org/W7163136233","doi":"https://doi.org/10.48550/arxiv.2606.01509"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.01509","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.01509","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.01509","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137621457","display_name":"Heng Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Heng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113264652","display_name":"Zilei Shao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shao, Zilei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137672622","display_name":"Guy Van den Broeck","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Broeck, Guy Van den","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5055376526","display_name":"Zhe Zeng","orcid":"https://orcid.org/0000-0002-0528-7069"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Zhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.325300008058548,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.325300008058548,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10064","display_name":"Complex Network Analysis Techniques","score":0.13570000231266022,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.12240000069141388,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/routing","display_name":"Routing (electronic design automation)","score":0.6766999959945679},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.6355999708175659},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.534500002861023},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5029000043869019},{"id":"https://openalex.org/keywords/cardinality","display_name":"Cardinality (data modeling)","score":0.47839999198913574},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.4690999984741211},{"id":"https://openalex.org/keywords/approximate-inference","display_name":"Approximate inference","score":0.36320000886917114},{"id":"https://openalex.org/keywords/model-selection","display_name":"Model selection","score":0.3321000039577484}],"concepts":[{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.6766999959945679},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6355999708175659},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5662000179290771},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.534500002861023},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5029000043869019},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4819999933242798},{"id":"https://openalex.org/C87117476","wikidata":"https://www.wikidata.org/wiki/Q362383","display_name":"Cardinality (data modeling)","level":2,"score":0.47839999198913574},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.4690999984741211},{"id":"https://openalex.org/C2777472644","wikidata":"https://www.wikidata.org/wiki/Q16968992","display_name":"Approximate inference","level":3,"score":0.36320000886917114},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.3321000039577484},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32910001277923584},{"id":"https://openalex.org/C76522221","wikidata":"https://www.wikidata.org/wiki/Q5035396","display_name":"Multipath routing","level":5,"score":0.3255000114440918},{"id":"https://openalex.org/C204948658","wikidata":"https://www.wikidata.org/wiki/Q1119410","display_name":"Static routing","level":4,"score":0.3107999861240387},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.30880001187324524},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.29829999804496765},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.2928999960422516},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.27730000019073486},{"id":"https://openalex.org/C2909817571","wikidata":"https://www.wikidata.org/wiki/Q310386","display_name":"Stizostedion","level":3,"score":0.260699987411499},{"id":"https://openalex.org/C202615002","wikidata":"https://www.wikidata.org/wiki/Q783507","display_name":"Differentiable function","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C184896649","wikidata":"https://www.wikidata.org/wiki/Q290066","display_name":"Routing table","level":4,"score":0.2538999915122986},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.25360000133514404}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.01509","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.01509","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.01509","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.01509","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"models":[2,17,48],"scale":[3],"by":[4],"activating":[5],"only":[6],"a":[7,36,43,52,96,107],"small":[8],"subset":[9,67],"of":[10],"experts":[11],"per":[12,129],"token.":[13,130],"However,":[14],"training":[15,113],"such":[16],"remains":[18,35],"challenging":[19],"because":[20],"top-$k$":[21],"routing":[22,45,60,109,118,150],"is":[23],"discrete":[24,66],"and":[25,58,83,114,133,149],"non-differentiable,":[26],"requiring":[27],"gradient":[28],"estimators":[29],"for":[30,99],"expert":[31,49,56,127,147],"selection":[32,50],"whose":[33],"design":[34],"central":[37],"open":[38],"problem.":[39],"We":[40,69],"introduce":[41],"ProbMoE,":[42],"probabilistic":[44,62],"framework":[46],"that":[47],"as":[51,61,95],"distribution":[53],"over":[54],"cardinality-constrained":[55],"subsets":[57,78],"formulates":[59],"inference":[63,115],"in":[64,79],"this":[65],"space.":[68],"first":[70],"propose":[71],"ProbMoE":[72,103,136,152],"Exact-$k$":[73,137],"routing,":[74],"which":[75],"samples":[76],"$k$-expert":[77],"the":[80,84,100,117,121],"forward":[81],"pass,":[82],"backward":[85],"pass":[86],"uses":[87],"gradients":[88],"through":[89],"each":[90],"expert's":[91],"exact":[92],"marginal":[93],"probability":[94],"tractable":[97],"surrogate":[98],"true":[101],"gradient.":[102],"naturally":[104],"generalizes":[105],"to":[106,120,142],"dynamic-$k$":[108],"setting,":[110],"where":[111],"both":[112],"constrain":[116],"cardinality":[119],"same":[122],"predefined":[123],"range,":[124],"allowing":[125],"adaptive":[126],"allocation":[128],"Across":[131],"benchmarks":[132],"model":[134],"backbones,":[135],"achieves":[138,154],"strong":[139],"performance":[140,156],"compared":[141],"competitive":[143],"baselines,":[144],"with":[145,157],"improved":[146],"utilization":[148],"diversity;":[151],"Dynamic-$k$":[153],"comparable":[155],"fewer":[158],"activated":[159],"experts.":[160]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-03T00:00:00"}