{"id":"https://openalex.org/W7165186692","doi":"https://doi.org/10.48550/arxiv.2606.18283","title":"Gaussian Mixture Attention: Linear-Time Sequence Mixing via Probabilistic Latent Routing","display_name":"Gaussian Mixture Attention: Linear-Time Sequence Mixing via Probabilistic Latent Routing","publication_year":2026,"publication_date":"2026-06-09","ids":{"openalex":"https://openalex.org/W7165186692","doi":"https://doi.org/10.48550/arxiv.2606.18283"},"language":"en","primary_location":{"id":"pmh:oai:HAL:hal-05651082v1","is_oa":true,"landing_page_url":"https://hal.science/hal-05651082","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2026","raw_type":"info:eu-repo/semantics/preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://hal.science/hal-05651082","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5126420301","display_name":"Y Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Yongchao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138939054","display_name":"Hassan Raza","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Raza, Hassan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.4634999930858612,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.4634999930858612,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.17560000717639923,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.048700001090765,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.6858999729156494},{"id":"https://openalex.org/keywords/gaussian","display_name":"Gaussian","score":0.5350000262260437},{"id":"https://openalex.org/keywords/mixture-model","display_name":"Mixture model","score":0.5051000118255615},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.4507000148296356},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.43650001287460327},{"id":"https://openalex.org/keywords/associative-property","display_name":"Associative property","score":0.3797000050544739},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.37459999322891235},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.37310001254081726},{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.37310001254081726}],"concepts":[{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6858999729156494},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5394999980926514},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.5350000262260437},{"id":"https://openalex.org/C61224824","wikidata":"https://www.wikidata.org/wiki/Q2260434","display_name":"Mixture model","level":2,"score":0.5051000118255615},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.4507000148296356},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.447299987077713},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.43650001287460327},{"id":"https://openalex.org/C159423971","wikidata":"https://www.wikidata.org/wiki/Q177251","display_name":"Associative property","level":2,"score":0.3797000050544739},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.37459999322891235},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.37310001254081726},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.37310001254081726},{"id":"https://openalex.org/C61326573","wikidata":"https://www.wikidata.org/wiki/Q1496376","display_name":"Gaussian process","level":3,"score":0.3596000075340271},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.353300005197525},{"id":"https://openalex.org/C202615002","wikidata":"https://www.wikidata.org/wiki/Q783507","display_name":"Differentiable function","level":2,"score":0.3353999853134155},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.3246000111103058},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.3237999975681305},{"id":"https://openalex.org/C51167844","wikidata":"https://www.wikidata.org/wiki/Q4422623","display_name":"Latent variable","level":2,"score":0.32019999623298645},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.31540000438690186},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.31360000371932983},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3059999942779541},{"id":"https://openalex.org/C138777275","wikidata":"https://www.wikidata.org/wiki/Q6884054","display_name":"Mixing (physics)","level":2,"score":0.30570000410079956},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.30250000953674316},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.29339998960494995},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.27639999985694885},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.26260000467300415},{"id":"https://openalex.org/C184720557","wikidata":"https://www.wikidata.org/wiki/Q7825049","display_name":"Topology (electrical circuits)","level":2,"score":0.25780001282691956},{"id":"https://openalex.org/C81081738","wikidata":"https://www.wikidata.org/wiki/Q55542","display_name":"Lossless compression","level":3,"score":0.2574000060558319}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:HAL:hal-05651082v1","is_oa":true,"landing_page_url":"https://hal.science/hal-05651082","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2026","raw_type":"info:eu-repo/semantics/preprint"},{"id":"doi:10.48550/arxiv.2606.18283","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18283","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:HAL:hal-05651082v1","is_oa":true,"landing_page_url":"https://hal.science/hal-05651082","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2026","raw_type":"info:eu-repo/semantics/preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"dense":[1],"token-to-token":[2],"interaction":[3],"pattern":[4],"of":[5,83,120,127,189],"standard":[6],"dot-product":[7],"attention":[8,172,222],"remains":[9,177],"a":[10,26,55,75,207,216],"central":[11],"bottleneck":[12],"in":[13,184],"scaling":[14,155],"Transformer":[15],"architectures":[16],"to":[17,50],"long":[18],"contexts.":[19],"We":[20,114],"introduce":[21],"\\textbf{Gaussian":[22],"Mixture":[23],"Attention":[24],"(GMA)},":[25],"probabilistic":[27],"attention-style":[28,160,212],"sequence":[29],"mixer":[30],"that":[31],"replaces":[32],"explicit":[33],"pairwise":[34],"query--key":[35],"comparison":[36],"with":[37,159,200],"routing":[38,58,145],"through":[39],"$K$":[40],"learned":[41,190],"Gaussian":[42,129],"mixture":[43,130],"components.":[44],"Queries":[45],"and":[46,72,95,117,132,143,156,182,197],"keys":[47],"are":[48,69],"mapped":[49],"posterior":[51],"\\textit{responsibility}":[52],"vectors":[53],"over":[54,169],"shared":[56],"latent":[57,77],"space;":[59],"their":[60],"overlap":[61],"defines":[62],"an":[63,123],"implicit":[64],"responsibility-space":[65],"affinity,":[66],"while":[67,165],"values":[68],"written":[70],"into":[71],"read":[73],"from":[74],"$K$-slot":[76],"memory.":[78],"By":[79],"exploiting":[80],"the":[81,89,128,150,185],"associativity":[82],"matrix":[84,94],"multiplication,":[85],"GMA":[86,148,167,205],"avoids":[87],"materializing":[88],"induced":[90],"$N\\times":[91],"N$":[92],"affinity":[93,141],"instead":[96],"uses":[97],"two":[98],"responsibility":[99],"matrices":[100],"whose":[101],"dominant":[102],"activation":[103],"storage":[104],"scales":[105],"as":[106,206],"$\\mathcal{O}(NK)$":[107],"rather":[108,214],"than":[109,215],"$\\mathcal{O}(N^2)$":[110],"for":[111,219],"fixed":[112],"$K$.":[113],"formulate":[115],"bidirectional":[116],"causal":[118,166,180],"variants":[119,173],"GMA,":[121],"provide":[122],"end-to-end":[124],"differentiable":[125],"parameterization":[126],"components,":[131],"analyze":[133],"its":[134],"responsibility-modulated":[135],"gradient":[136],"structure,":[137],"constrained":[138],"non-negative":[139],"low-rank":[140],"interpretation,":[142],"local":[144],"stability.":[146],"Empirically,":[147],"exhibits":[149],"intended":[151],"fixed-$K$":[152,210],"linear":[153],"memory":[154],"is":[157],"competitive":[158],"baselines":[161],"on":[162,174],"long-context":[163],"classification,":[164],"improves":[168],"tested":[170],"linear/random-feature":[171],"WikiText-103":[175],"but":[176],"behind":[178],"optimized":[179,220],"SDPA":[181],"Mamba":[183],"current":[186],"implementation.":[187],"Analysis":[188],"responsibilities":[191],"further":[192],"shows":[193],"broad":[194],"component":[195],"usage":[196],"moderate":[198],"alignment":[199],"surface-form":[201],"token":[202],"categories,":[203],"supporting":[204],"probabilistic,":[208],"interpretable,":[209],"linear-time":[211],"alternative":[213],"universal":[217],"replacement":[218],"softmax":[221],"or":[223],"state-space":[224],"models.":[225]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-19T00:00:00"}