{"id":"https://openalex.org/W7214808245","doi":"https://doi.org/10.48550/arxiv.2609.35349","title":"Quasi Linear Kernel Attention with Infinite Capacity","display_name":"Quasi Linear Kernel Attention with Infinite Capacity","publication_year":2026,"publication_date":"2026-09-28","ids":{"openalex":"https://openalex.org/W7214808245","doi":"https://doi.org/10.48550/arxiv.2609.35349"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.35349","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35349","pdf_url":"https://arxiv.org/pdf/2609.35349","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.35349","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114729853","display_name":"Nicolaj Rux","orcid":"https://orcid.org/0009-0000-1299-0580","observed_orcids":["https://orcid.org/0009-0000-1299-0580"]},"institutions":[{"id":"https://openalex.org/I2610724","display_name":"Chemnitz University of Technology","ror":"https://ror.org/00a208s56","country_code":"DE","type":"education","lineage":["https://openalex.org/I2610724"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Rux, Nicolaj","raw_affiliation_strings":["Faculty of Mathematics , Chemnitz University of Technology , 09126 Chemnitz , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Mathematics , Chemnitz University of Technology , 09126 Chemnitz , Germany","institution_ids":["https://openalex.org/I2610724"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007073244","display_name":"Johannes Hertrich","orcid":"https://orcid.org/0000-0003-4433-8604","observed_orcids":["https://orcid.org/0000-0003-4433-8604"]},"institutions":[{"id":"https://openalex.org/I74656192","display_name":"University of G\u00f6ttingen","ror":"https://ror.org/01y9bpm73","country_code":"DE","type":"education","lineage":["https://openalex.org/I74656192"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Hertrich, Johannes","raw_affiliation_strings":["Institute of Computer Science , University of G\u00f6ttingen , 37073 G\u00f6ttingen , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computer Science , University of G\u00f6ttingen , 37073 G\u00f6ttingen , Germany","institution_ids":["https://openalex.org/I74656192"]}]},{"author_position":"last","author":{"id":null,"display_name":"Neumayer, Sebastian","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2610724","display_name":"Chemnitz University of Technology","ror":"https://ror.org/00a208s56","country_code":"DE","type":"education","lineage":["https://openalex.org/I2610724"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Neumayer, Sebastian","raw_affiliation_strings":["Faculty of Mathematics , Chemnitz University of Technology , 09126 Chemnitz , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Mathematics , Chemnitz University of Technology , 09126 Chemnitz , Germany","institution_ids":["https://openalex.org/I2610724"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7695000171661377,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7695000171661377,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0364999994635582,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.03539999946951866,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/expressivity","display_name":"expressivity","score":0.5360000133514404},{"id":"https://openalex.org/keywords/softmax-attention","display_name":"softmax attention","score":0.6899999976158142},{"id":"https://openalex.org/keywords/additive-kernels","display_name":"additive kernels","score":0.6430000066757202},{"id":"https://openalex.org/keywords/quadratic-complexity","display_name":"quadratic complexity","score":0.4830000102519989}],"concepts":[{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.6904000043869019},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.6549999713897705},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5275999903678894},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.5203999876976013},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.45159998536109924},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.44530001282691956},{"id":"https://openalex.org/C97937538","wikidata":"https://www.wikidata.org/wiki/Q199691","display_name":"Laplace transform","level":2,"score":0.4318000078201294},{"id":"https://openalex.org/C151376022","wikidata":"https://www.wikidata.org/wiki/Q168698","display_name":"Exponential function","level":2,"score":0.40720000863075256},{"id":"https://openalex.org/C90119067","wikidata":"https://www.wikidata.org/wiki/Q43260","display_name":"Polynomial","level":2,"score":0.3831999897956848},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.38040000200271606},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.37929999828338623},{"id":"https://openalex.org/C195956108","wikidata":"https://www.wikidata.org/wiki/Q7268362","display_name":"Quadratic growth","level":2,"score":0.37049999833106995},{"id":"https://openalex.org/C122280245","wikidata":"https://www.wikidata.org/wiki/Q620622","display_name":"Kernel method","level":3,"score":0.3427000045776367},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3370000123977661},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.335999995470047},{"id":"https://openalex.org/C2777759810","wikidata":"https://www.wikidata.org/wiki/Q149316","display_name":"Lemma (botany)","level":3,"score":0.3287999927997589},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.32739999890327454},{"id":"https://openalex.org/C163175372","wikidata":"https://www.wikidata.org/wiki/Q3339222","display_name":"Linear model","level":2,"score":0.32179999351501465},{"id":"https://openalex.org/C160446489","wikidata":"https://www.wikidata.org/wiki/Q7226642","display_name":"Polynomial kernel","level":4,"score":0.3199999928474426},{"id":"https://openalex.org/C203223496","wikidata":"https://www.wikidata.org/wiki/Q4681344","display_name":"Additive model","level":2,"score":0.30329999327659607},{"id":"https://openalex.org/C199163554","wikidata":"https://www.wikidata.org/wiki/Q1681619","display_name":"Univariate","level":3,"score":0.28790000081062317},{"id":"https://openalex.org/C10390562","wikidata":"https://www.wikidata.org/wiki/Q581809","display_name":"Spline (mechanical)","level":2,"score":0.27889999747276306},{"id":"https://openalex.org/C195906000","wikidata":"https://www.wikidata.org/wiki/Q1191722","display_name":"Matrix exponential","level":3,"score":0.2782999873161316},{"id":"https://openalex.org/C194648359","wikidata":"https://www.wikidata.org/wiki/Q3318054","display_name":"Generalized additive model","level":2,"score":0.27239999175071716},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.2687999904155731},{"id":"https://openalex.org/C6802819","wikidata":"https://www.wikidata.org/wiki/Q1072174","display_name":"Linear system","level":2,"score":0.2639000117778778},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.2628999948501587},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.26269999146461487},{"id":"https://openalex.org/C163834973","wikidata":"https://www.wikidata.org/wiki/Q2004891","display_name":"Numerical linear algebra","level":3,"score":0.25209999084472656}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.35349","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35349","pdf_url":"https://arxiv.org/pdf/2609.35349","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.35349","doi":"https://doi.org/10.48550/arxiv.2609.35349","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.35349","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.35349","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35349","pdf_url":"https://arxiv.org/pdf/2609.35349","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1615899948","display_name":"Variationelle Gradientenfl\u00fcsse auf Wahrscheinlichkeitsr\u00e4umen und Generative Modelle f\u00fcr Bayessche Inverse Probleme in der Bildverarbeitung","funder_award_id":"530824055","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320328735","display_name":"Freistaat Sachsen","ror":"https://ror.org/02hyyn853"},{"id":"https://openalex.org/F4320338080","display_name":"European Social Fund","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214808245.pdf","grobid_xml":"https://content.openalex.org/works/W7214808245.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W1988224016","https://openalex.org/W2004026774","https://openalex.org/W2059201851","https://openalex.org/W2128703518","https://openalex.org/W2500592145","https://openalex.org/W2805334229","https://openalex.org/W2970100546","https://openalex.org/W3015468748","https://openalex.org/W3023320044","https://openalex.org/W3033529678","https://openalex.org/W3085139254","https://openalex.org/W3141455185","https://openalex.org/W3173365702","https://openalex.org/W4386576685","https://openalex.org/W4401306886","https://openalex.org/W4404488434","https://openalex.org/W4414847473","https://openalex.org/W4415795816","https://openalex.org/W4417280654","https://openalex.org/W7090815571","https://openalex.org/W7133227460"],"related_works":["https://openalex.org/W7160393803","https://openalex.org/W7108733241","https://openalex.org/W7114773938","https://openalex.org/W7108741320","https://openalex.org/W7166507899","https://openalex.org/W7108616313","https://openalex.org/W7169835299","https://openalex.org/W7223324510","https://openalex.org/W4417280654","https://openalex.org/W7221787703"],"abstract_inverted_index":{"The":[0],"evaluation":[1],"cost":[2],"of":[3,38],"transformers":[4],"with":[5,10,20],"softmax":[6,19,148],"attention":[7,14,39,63],"scales":[8],"quadratically":[9],"sequence":[11,58],"length.":[12],"Kernel":[13],"addresses":[15],"this":[16,27],"by":[17],"replacing":[18],"a":[21,50,108],"more":[22],"general":[23],"kernel":[24],"function.":[25],"In":[26,89],"paper,":[28],"we":[29,48,110,137],"aim":[30],"to":[31],"identify":[32],"kernels":[33,80,113,141],"that":[34,78,124],"retain":[35],"the":[36,56,62,67],"expressivity":[37],"while":[40,129],"enabling":[41],"quasi":[42,92,131],"linear":[43,93,132],"computation.":[44],"To":[45],"quantify":[46],"expressivity,":[47],"introduce":[49],"capacity":[51,71,128],"for":[52,60,152],"each":[53],"kernel,":[54],"measuring":[55],"maximum":[57],"length":[59],"which":[61],"matrix":[64],"can":[65],"approximate":[66],"identity.":[68],"A":[69],"higher":[70],"thus":[72],"indicates":[73],"greater":[74],"expressivity.":[75],"We":[76,122],"show":[77],"expressive":[79],"like":[81],"softmax,":[82],"Gauss,":[83],"and":[84,118,143],"Laplace":[85],"have":[86],"infinite":[87,127],"capacity.":[88,106],"contrast,":[90],"common":[91],"kernels,":[94],"such":[95],"as":[96],"those":[97],"derived":[98],"from":[99,115],"finite":[100,105],"dimensional":[101],"feature":[102],"maps,":[103],"exhibit":[104],"As":[107],"solution,":[109],"propose":[111],"additive":[112,139],"constructed":[114],"univariate":[116],"spline":[117],"polynomial":[119],"exponential":[120],"kernels.":[121],"prove":[123],"these":[125],"maintain":[126],"allowing":[130],"computation":[133],"via":[134],"sorting.":[135],"Finally,":[136],"implement":[138],"sorting":[140],"efficiently":[142],"benchmark":[144],"them":[145],"against":[146],"modern":[147],"backends,":[149],"demonstrating":[150],"advantages":[151],"long":[153],"sequences.":[154]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-30T00:00:00"}