{"id":"https://openalex.org/W7225465739","doi":"https://doi.org/10.5281/zenodo.24032653","title":"Modular Connections: The Connections Between Residual Streams Can Themselves Be Modular Models","display_name":"Modular Connections: The Connections Between Residual Streams Can Themselves Be Modular Models","publication_year":2026,"publication_date":"2026-11-06","ids":{"openalex":"https://openalex.org/W7225465739","doi":"https://doi.org/10.5281/zenodo.24032653"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.24032653","doi":"https://doi.org/10.5281/zenodo.24032653","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24032653","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.24032653","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5161182753","display_name":"Yue Su","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Su, Yue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5161182753"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.47749999165534973,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.47749999165534973,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.39719998836517334,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.035599999129772186,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mixture-of-experts","display_name":"mixture of experts","score":0.8769999742507935},{"id":"https://openalex.org/keywords/deconvolution","display_name":"deconvolution","score":0.7739999890327454},{"id":"https://openalex.org/keywords/expert-routing","display_name":"expert routing","score":0.6100000143051147},{"id":"https://openalex.org/keywords/deepseek-v4-flash","display_name":"DeepSeek-V4-Flash","score":0.7910000085830688},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.46000000834465027},{"id":"https://openalex.org/keywords/inference-acceleration","display_name":"inference acceleration","score":0.3880000114440918}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7732999920845032},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6470000147819519},{"id":"https://openalex.org/C42090638","wikidata":"https://www.wikidata.org/wiki/Q4048907","display_name":"STREAMS","level":2,"score":0.5612999796867371},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.5444999933242798},{"id":"https://openalex.org/C89198739","wikidata":"https://www.wikidata.org/wiki/Q3079880","display_name":"Data stream mining","level":2,"score":0.5073000192642212},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.4251999855041504},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.4011000096797943},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.38089999556541443},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.38019999861717224},{"id":"https://openalex.org/C118930307","wikidata":"https://www.wikidata.org/wiki/Q600590","display_name":"Tuple","level":2,"score":0.3637999892234802},{"id":"https://openalex.org/C147494362","wikidata":"https://www.wikidata.org/wiki/Q2078905","display_name":"Troubleshooting","level":2,"score":0.3492000102996826},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.328000009059906},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.32760000228881836},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.3231000006198883},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.3224000036716461},{"id":"https://openalex.org/C45347329","wikidata":"https://www.wikidata.org/wiki/Q5166604","display_name":"Convolution (computer science)","level":3,"score":0.30790001153945923},{"id":"https://openalex.org/C123745756","wikidata":"https://www.wikidata.org/wiki/Q1665949","display_name":"Interconnection","level":2,"score":0.2971999943256378},{"id":"https://openalex.org/C34736171","wikidata":"https://www.wikidata.org/wiki/Q918333","display_name":"Preprocessor","level":2,"score":0.29510000348091125},{"id":"https://openalex.org/C43711488","wikidata":"https://www.wikidata.org/wiki/Q7534783","display_name":"Skew","level":2,"score":0.2874999940395355},{"id":"https://openalex.org/C56951928","wikidata":"https://www.wikidata.org/wiki/Q3539213","display_name":"Trimming","level":2,"score":0.2824999988079071},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.2793000042438507},{"id":"https://openalex.org/C21308566","wikidata":"https://www.wikidata.org/wiki/Q7169365","display_name":"Permutation (music)","level":2,"score":0.27480000257492065},{"id":"https://openalex.org/C96146094","wikidata":"https://www.wikidata.org/wiki/Q609057","display_name":"Unification","level":2,"score":0.274399995803833},{"id":"https://openalex.org/C54732982","wikidata":"https://www.wikidata.org/wiki/Q1415345","display_name":"Modulo","level":2,"score":0.2734000086784363},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2685000002384186},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.26649999618530273},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.2635999917984009},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2632000148296356},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.2603999972343445}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.24032653","doi":"https://doi.org/10.5281/zenodo.24032653","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24032653","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.24032654","doi":"https://doi.org/10.5281/zenodo.24032654","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24032654","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.24032653","doi":"https://doi.org/10.5281/zenodo.24032653","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24032653","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7211886651","https://openalex.org/W7225158809","https://openalex.org/W7138028481","https://openalex.org/W7163361253","https://openalex.org/W7169497073","https://openalex.org/W7202190163","https://openalex.org/W7169854094","https://openalex.org/W7154992751","https://openalex.org/W7133351021","https://openalex.org/W4407759118"],"abstract_inverted_index":{"Manifold-Constrained":[0],"Hyper-Connections":[1],"(mHC)":[2],"widen":[3],"the":[4,35,42,69,144,171,196,216],"residual":[5],"stream":[6],"of":[7,71,78,93,107,113,126,147,180,219,227],"a":[8,52,55,64,76,105,111,123,168],"Transformer":[9],"into":[10],"N":[11,43],"parallel":[12],"streams":[13,24,29,36,44],"and,":[14],"at":[15,229],"every":[16,89],"sublayer,":[17],"generate":[18],"per-token":[19,141,154],"coefficients":[20],"that":[21,40,179],"decide":[22],"which":[23,28],"to":[25,30,104,225,235],"read":[26],"from,":[27],"write":[31,95],"to,":[32],"and":[33,54,60,63,96,99,115,187,203],"how":[34],"mix.":[37],"I":[38,84],"show":[39],"once":[41,194],"are":[45,50,74,149],"interleaved,":[46],"these":[47],"three":[48],"operations":[49],"exactly":[51],"convolution":[53],"deconvolution":[56],"with":[57,118,139,240],"kernel":[58],"size":[59],"stride":[61],"N,":[62],"sliding":[65],"N\u00d7N":[66],"map,":[67],"so":[68],"connections":[70],"each":[72,100],"token":[73,101],"simply":[75],"set":[77],"kernels.":[79],"Building":[80],"on":[81,132,167,182,191],"this":[82],"view,":[83],"propose":[85],"Modular":[86],"Connections":[87],"(MoC):":[88],"sublayer":[90,138],"keeps":[91],"banks":[92],"read,":[94],"mixing":[97,145],"experts,":[98,114],"is":[102,175],"routed":[103],"few":[106],"them,":[108],"as":[109],"in":[110],"mixture":[112],"combines":[116],"them":[117],"weights.":[119],"This":[120],"design":[121],"matches":[122],"recent":[124],"analysis":[125],"DeepSeek-V4-Flash,":[127],"whose":[128],"trained":[129],"mHC":[130,181,228],"settles":[131],"fairly":[133],"fixed":[134],"connection":[135],"patterns":[136],"per":[137],"little":[140],"variation.":[142],"Because":[143],"matrices":[146],"MoC":[148,172,220],"expert":[150,208],"parameters":[151],"rather":[152],"than":[153,178],"outputs,":[155],"their":[156],"Sinkhorn":[157],"normalization":[158],"can":[159,205],"be":[160,206],"precomputed":[161],"after":[162],"training.":[163],"In":[164,211],"my":[165],"measurements":[166],"1.36B-parameter":[169],"model,":[170],"forward":[173],"pass":[174],"2.2\u20133.2\u00d7":[176],"faster":[177,190],"single":[183],"sequences":[184],"without":[185],"compilation":[186],"about":[188,200],"1.5\u00d7":[189],"short":[192],"inputs":[193],"compiled,":[195],"training":[197],"step":[198,233,236],"compiles":[199],"16\u00d7":[201],"faster,":[202],"routing":[204],"inspected":[207],"by":[209],"expert.":[210],"an":[212],"ongoing":[213],"49B-token":[214],"run,":[215],"validation":[217],"bits-per-byte":[218],"has":[221],"stayed":[222],"within":[223],"\u22120.0009":[224],"+0.0017":[226],"matching":[230],"steps":[231],"from":[232],"2,000":[234],"6,360.":[237],"Web":[238],"version":[239],"interactive":[241],"animations:":[242],"https://selen-suyue.github.io/modular-connections/en/":[243]},"counts_by_year":[],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2026-11-08T00:00:00"}