{"id":"https://openalex.org/W7138903718","doi":"https://doi.org/10.48550/arxiv.2603.17946","title":"CARE: Covariance-Aware and Rank-Enhanced Decomposition for Enabling Multi-Head Latent Attention","display_name":"CARE: Covariance-Aware and Rank-Enhanced Decomposition for Enabling Multi-Head Latent Attention","publication_year":2026,"publication_date":"2026-03-18","ids":{"openalex":"https://openalex.org/W7138903718","doi":"https://doi.org/10.48550/arxiv.2603.17946"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.17946","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.17946","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.17946","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102686304","display_name":"Zhongzhu Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Zhongzhu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120826418","display_name":"Fengxiang Bie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bie, Fengxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130142173","display_name":"Ziyan Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Ziyan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129783735","display_name":"Zhenyu Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Zhenyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129962345","display_name":"Yibo Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yibo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051789410","display_name":"Junxiong Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Junxiong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130026272","display_name":"Ben Athiwaratkun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Athiwaratkun, Ben","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129896826","display_name":"Xiaoxia Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Xiaoxia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129858981","display_name":"Shuaiwen Leon Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Shuaiwen Leon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.1956000030040741,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.1956000030040741,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.13030000030994415,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.11169999837875366,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.7441999912261963},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.6460000276565552},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.6036999821662903},{"id":"https://openalex.org/keywords/covariance","display_name":"Covariance","score":0.585099995136261},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.5835000276565552},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5455999970436096},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5450999736785889},{"id":"https://openalex.org/keywords/decomposition","display_name":"Decomposition","score":0.44530001282691956}],"concepts":[{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.7441999912261963},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.6460000276565552},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6453999876976013},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.6036999821662903},{"id":"https://openalex.org/C178650346","wikidata":"https://www.wikidata.org/wiki/Q201984","display_name":"Covariance","level":2,"score":0.585099995136261},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.5835000276565552},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5455999970436096},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5450999736785889},{"id":"https://openalex.org/C124681953","wikidata":"https://www.wikidata.org/wiki/Q339062","display_name":"Decomposition","level":2,"score":0.44530001282691956},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4018000066280365},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.38519999384880066},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3675000071525574},{"id":"https://openalex.org/C22789450","wikidata":"https://www.wikidata.org/wiki/Q420904","display_name":"Singular value decomposition","level":2,"score":0.3172000050544739},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.30489999055862427},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3012000024318695},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.289900004863739},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2870999872684479},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.2799000144004822},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.2703000009059906},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2637999951839447},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.2624000012874603},{"id":"https://openalex.org/C2993807640","wikidata":"https://www.wikidata.org/wiki/Q103709453","display_name":"Attention network","level":2,"score":0.2572000026702881},{"id":"https://openalex.org/C137800194","wikidata":"https://www.wikidata.org/wiki/Q11713455","display_name":"Interpolation (computer graphics)","level":3,"score":0.25200000405311584},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.25049999356269836}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.17946","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.17946","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.17946","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.17946","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Converting":[0],"pretrained":[1],"attention":[2,7,12,79],"modules":[3],"such":[4],"as":[5],"grouped-query":[6],"(GQA)":[8],"into":[9],"multi-head":[10],"latent":[11],"(MLA)":[13],"can":[14],"improve":[15],"expressivity":[16],"without":[17],"increasing":[18],"KV-cache":[19,161],"cost,":[20],"making":[21],"it":[22,140],"attractive":[23],"for":[24],"efficient":[25],"inference.":[26],"However,":[27],"many":[28],"practical":[29],"conversion":[30,92],"baselines":[31],"rely":[32],"on":[33,46,55,171],"weight-only":[34],"low-rank":[35],"approximations":[36],"(e.g.,":[37],"SVD-style":[38],"initializations)":[39],"and":[40,68,77,142,151,173,182],"uniform":[41,70],"rank":[42,71],"allocation.":[43],"They":[44],"focus":[45],"minimizing":[47],"the":[48,63,109,112,119,148,155,160,203],"difference":[49],"between":[50],"weight":[51],"matrices":[52],"rather":[53,116],"than":[54,117],"how":[56],"those":[57],"weights":[58],"affect":[59],"input":[60,114],"activations,":[61,67],"ignore":[62],"covariance":[64],"structure":[65],"of":[66],"enforce":[69],"across":[72,130],"layers,":[73],"causing":[74],"activation":[75],"drift":[76],"degraded":[78],"fidelity.":[80],"To":[81],"address":[82],"these":[83],"issues,":[84],"we":[85,200],"propose":[86],"CARE,":[87],"a":[88,95,126,167,195],"Covariance-Aware,":[89],"Rank-Enhanced":[90],"MLA":[91,156],"pipeline":[93],"under":[94],"fixed":[96,127],"KV":[97,128,192],"width.":[98],"CARE":[99],"introduces":[100],"three":[101],"key":[102],"steps:":[103],"(i)":[104],"activation-preserving":[105],"factorization,":[106],"which":[107,124,146],"aligns":[108],"approximation":[110],"with":[111],"actual":[113],"activations":[115],"just":[118],"weights;":[120],"(ii)":[121],"adjusted-rank":[122],"allocation,":[123],"spreads":[125],"budget":[129],"layers":[131,137],"by":[132,178,186],"giving":[133],"more":[134],"capacity":[135],"to":[136,153,180,188],"that":[138],"need":[139],"most;":[141],"(iii)":[143],"KV-parity":[144],"mapping,":[145],"reparameterizes":[147],"converted":[149],"K":[150],"V":[152],"fit":[154],"format":[157],"while":[158],"keeping":[159],"size":[162],"unchanged.":[163],"Our":[164],"method":[165],"outperforms":[166],"uniform-rank":[168],"SVD":[169],"baseline":[170],"Qwen3-4B/30B-A3B-Instruct-2507":[172],"Llama-3.1-8B/70B-Instruct,":[174],"reducing":[175],"one-shot":[176],"perplexity":[177],"up":[179,187],"215x":[181],"improving":[183],"mean":[184],"accuracy":[185],"1.70x":[189],"at":[190],"matched":[191],"budgets.":[193],"With":[194],"brief":[196],"post-SVD":[197],"healing":[198],"fine-tune,":[199],"fully":[201],"recover":[202],"original":[204],"model's":[205],"accuracy.":[206]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-20T00:00:00"}