{"id":"https://openalex.org/W7164894731","doi":"https://doi.org/10.48550/arxiv.2606.16310","title":"QK-Normed MLA: QK normalization without full key caching","display_name":"QK-Normed MLA: QK normalization without full key caching","publication_year":2026,"publication_date":"2026-06-15","ids":{"openalex":"https://openalex.org/W7164894731","doi":"https://doi.org/10.48550/arxiv.2606.16310"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.16310","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.16310","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.16310","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138748388","display_name":"Yizhou Han","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Yizhou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138704587","display_name":"Yao Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138693263","display_name":"Jun Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Jun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138720354","display_name":"Longfei Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Longfei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5109298191","display_name":"Ruoyu Sun","orcid":"https://orcid.org/0000-0001-7261-8377"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Ruoyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.1589999943971634,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.1589999943971634,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.11270000040531158,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.10180000215768814,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/affine-transformation","display_name":"Affine transformation","score":0.5415999889373779},{"id":"https://openalex.org/keywords/normalization","display_name":"Normalization (sociology)","score":0.522599995136261},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.4426000118255615},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.4284999966621399},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.42340001463890076},{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.39169999957084656}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6179999709129333},{"id":"https://openalex.org/C92757383","wikidata":"https://www.wikidata.org/wiki/Q382497","display_name":"Affine transformation","level":2,"score":0.5415999889373779},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.522599995136261},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.49799999594688416},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.4426000118255615},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.4284999966621399},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.42340001463890076},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.4142000079154968},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.39430001378059387},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.39169999957084656},{"id":"https://openalex.org/C89128539","wikidata":"https://www.wikidata.org/wiki/Q1949963","display_name":"Statistic","level":2,"score":0.3797999918460846},{"id":"https://openalex.org/C19275194","wikidata":"https://www.wikidata.org/wiki/Q222903","display_name":"Multiplexing","level":2,"score":0.3671000003814697},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.3483999967575073},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.30559998750686646},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.29679998755455017},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.29649999737739563},{"id":"https://openalex.org/C178197554","wikidata":"https://www.wikidata.org/wiki/Q1099110","display_name":"Sufficient statistic","level":2,"score":0.2939000129699707},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.28859999775886536}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.16310","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.16310","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.16310","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.16310","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Query-key":[0],"(QK)":[1],"normalization":[2,168],"stabilizes":[3],"attention":[4],"by":[5,31],"controlling":[6],"the":[7,14,47,89,93],"scale":[8],"of":[9,37],"queries":[10],"and":[11,75,104,121,143],"keys":[12],"before":[13],"dot":[15],"product,":[16],"but":[17],"is":[18,60,110],"not":[19,64],"immediately":[20],"compatible":[21],"with":[22],"Multi-head":[23],"Latent":[24],"Attention":[25],"(MLA).":[26],"MLA":[27,90,138,174],"achieves":[28,139],"efficient":[29],"decoding":[30],"caching":[32],"low-dimensional":[33],"latent":[34,124],"states":[35],"instead":[36],"full":[38],"keys,":[39],"whereas":[40],"post-projection":[41,115],"QK":[42,116,148,167],"RMSNorm":[43,68,117],"appears":[44],"to":[45,98,113,134,161],"require":[46],"fully":[48],"projected":[49],"key":[50,95],"for":[51,132,173],"every":[52],"cached":[53],"token.":[54],"We":[55],"show":[56,154],"this":[57],"apparent":[58],"incompatibility":[59],"an":[61,65],"implementation":[62],"artifact,":[63],"architectural":[66],"constraint.":[67],"decomposes":[69],"into":[70,88],"a":[71,76,169],"static":[72,82],"affine":[73],"weight":[74,84],"dynamic":[77,94],"scalar":[78,101],"RMS":[79],"statistic.":[80],"The":[81,107],"key-side":[83],"can":[85],"be":[86],"absorbed":[87],"query-side":[91],"projection;":[92],"statistic":[96],"reduces":[97],"one":[99],"inverse-RMS":[100],"per":[102],"token":[103],"KV":[105],"group.":[106],"resulting":[108],"formulation":[109],"exactly":[111],"equivalent":[112],"explicit":[114],"in":[118],"exact":[119],"arithmetic":[120],"preserves":[122],"MLA's":[123],"decode":[125,152],"path.":[126],"In":[127],"our":[128],"400M":[129],"runs":[130],"trained":[131],"up":[133,160],"100B":[135],"tokens,":[136],"QK-Normed":[137],"lower":[140],"training":[141],"loss":[142],"better":[144],"downstream":[145],"accuracy":[146],"than":[147,156],"clipping,":[149],"while":[150],"H800":[151],"benchmarks":[153],"less":[155],"2%":[157],"latency":[158],"overhead":[159],"256k":[162],"context.":[163],"These":[164],"results":[165],"make":[166],"practical":[170],"stabilization":[171],"option":[172],"models":[175],"without":[176],"requiring":[177],"full-key":[178],"caching.":[179]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-17T00:00:00"}