{"id":"https://openalex.org/W7162681005","doi":"https://doi.org/10.48550/arxiv.2605.28769","title":"Multi-Mixer Models: Flexible Sequence Modeling with Shared Representations","display_name":"Multi-Mixer Models: Flexible Sequence Modeling with Shared Representations","publication_year":2026,"publication_date":"2026-05-27","ids":{"openalex":"https://openalex.org/W7162681005","doi":"https://doi.org/10.48550/arxiv.2605.28769"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.28769","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.28769","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.28769","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137285257","display_name":"Kevin Y. Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Kevin Y.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049077447","display_name":"Asher Trockman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Trockman, Asher","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103890688","display_name":"Ananda Theertha Suresh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Suresh, Ananda Theertha","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5001128596","display_name":"Ziteng Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Ziteng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.36730000376701355,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.36730000376701355,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.21359999477863312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.07940000295639038,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.6227999925613403},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5335999727249146},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.46209999918937683},{"id":"https://openalex.org/keywords/oryx","display_name":"Oryx","score":0.4406000077724457},{"id":"https://openalex.org/keywords/interleaving","display_name":"Interleaving","score":0.3617999851703644},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.352400004863739},{"id":"https://openalex.org/keywords/linear-model","display_name":"Linear model","score":0.3513000011444092},{"id":"https://openalex.org/keywords/constant","display_name":"Constant (computer programming)","score":0.3244999945163727}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6830000281333923},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6227999925613403},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5335999727249146},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.46209999918937683},{"id":"https://openalex.org/C2779536738","wikidata":"https://www.wikidata.org/wiki/Q658179","display_name":"Oryx","level":2,"score":0.4406000077724457},{"id":"https://openalex.org/C28034677","wikidata":"https://www.wikidata.org/wiki/Q17092530","display_name":"Interleaving","level":2,"score":0.3617999851703644},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.352400004863739},{"id":"https://openalex.org/C163175372","wikidata":"https://www.wikidata.org/wiki/Q3339222","display_name":"Linear model","level":2,"score":0.3513000011444092},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33739998936653137},{"id":"https://openalex.org/C2777027219","wikidata":"https://www.wikidata.org/wiki/Q1284190","display_name":"Constant (computer programming)","level":2,"score":0.3244999945163727},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.32260000705718994},{"id":"https://openalex.org/C28077239","wikidata":"https://www.wikidata.org/wiki/Q841728","display_name":"Linear logic","level":2,"score":0.32120001316070557},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.3183000087738037},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.3138999938964844},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.30709999799728394},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2973000109195709},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.28929999470710754},{"id":"https://openalex.org/C149629883","wikidata":"https://www.wikidata.org/wiki/Q660926","display_name":"Fraction (chemistry)","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.27059999108314514},{"id":"https://openalex.org/C2776809875","wikidata":"https://www.wikidata.org/wiki/Q1375963","display_name":"Converse","level":2,"score":0.25609999895095825},{"id":"https://openalex.org/C35651441","wikidata":"https://www.wikidata.org/wiki/Q625303","display_name":"Independence (probability theory)","level":2,"score":0.25049999356269836}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.28769","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.28769","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.28769","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.28769","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.5055971741676331,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Softmax":[0],"attention":[1,27,39,104,143,165,253,259],"is":[2],"the":[3,121,208,238,250],"cornerstone":[4],"of":[5,67,88,116,159,213,249],"modern":[6],"large":[7],"language":[8,227],"models,":[9,23,31],"but":[10],"its":[11,160,204],"memory":[12],"scales":[13],"linearly":[14],"and":[15,28,45,60,105,148,166,181,193,260,268],"compute":[16,44],"quadratically":[17],"with":[18,179],"sequence":[19],"length.":[20],"Linear":[21],"recurrent":[22,71,106,167,262],"such":[24],"as":[25,36,272],"linear":[26,43,70,149,261],"state":[29],"space":[30],"have":[32],"become":[33],"widely":[34],"studied":[35],"alternatives":[37],"to":[38,41,95,169,186,237],"due":[40],"their":[42,216],"constant":[46],"memory.":[47],"While":[48],"these":[49,97],"sub-quadratic":[50],"token":[51,122,191],"mixing":[52],"methods,":[53],"or":[54,82,102,200],"mixers,":[55,139,163],"achieve":[56],"promising":[57,274],"efficiency":[58],"gains":[59],"competitive":[61],"results":[62,256],"on":[63,76,225],"a":[64,113,127,133,194,245,273],"wide":[65],"range":[66],"benchmarks,":[68],"current":[69],"models":[72,263],"still":[73],"lag":[74],"behind":[75],"tasks":[77],"that":[78,93,130,258],"require":[79],"long-context":[80],"retrieval":[81,231],"in-context":[83],"learning.":[84],"A":[85],"growing":[86],"body":[87],"work":[89],"studies":[90],"hybrid":[91,118,128],"architectures":[92],"attempt":[94],"mitigate":[96],"trade-offs":[98],"by":[99,219],"statically":[100],"interleaving":[101],"merging":[103],"blocks.":[107],"In":[108],"this":[109],"work,":[110],"we":[111],"explore":[112],"new":[114],"axis":[115],"developing":[117],"models:":[119],"across":[120,162],"sequence.":[123],"We":[124,175],"propose":[125],"Oryx,":[126],"model":[129],"can,":[131],"throughout":[132],"sequence,":[134],"flexibly":[135],"switch":[136],"between":[137],"different":[138],"for":[140,144,151],"example":[141],"quadratic":[142],"rich":[145],"context":[146],"utilization":[147],"recurrences":[150],"efficient":[152],"generation.":[153],"Oryx":[154,197,214,233],"ties":[155],"at":[156,220],"least":[157,221],"90%":[158],"parameters":[161],"enabling":[164],"modes":[168],"operate":[170],"over":[171],"shared":[172],"internal":[173,266],"representations.":[174],"validate":[176],"our":[177],"design":[178],"Mamba-2":[180],"Gated":[182],"DeltaNet":[183],"variants,":[184],"up":[185],"1.4B":[187,209],"models.":[188],"Under":[189],"fixed":[190],"budgets":[192],"mixed-training":[195],"strategy,":[196],"achieves":[198,234],"comparable":[199,236],"better":[201],"performance":[202,235],"than":[203],"single-mixer":[205],"baselines.":[206],"At":[207],"scale,":[210],"all":[211],"instances":[212],"outperform":[215],"respective":[217],"baselines":[218],"0.7":[222],"percentage":[223],"points":[224],"averaged":[226],"modeling":[228],"tasks.":[229],"On":[230],"tasks,":[232],"Transformer":[239],"baseline":[240],"even":[241],"when":[242],"processing":[243],"only":[244],"tiny":[246],"fraction":[247],"(<10%)":[248],"tokens":[251],"in":[252],"mode.":[254],"These":[255],"suggest":[257],"can":[264],"share":[265],"representations,":[267],"motivate":[269],"sequence-axis":[270],"hybridization":[271],"direction.":[275]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-29T00:00:00"}