{"id":"https://openalex.org/W7169794564","doi":"https://doi.org/10.48550/arxiv.2607.15456","title":"Looped Latent Attention: Cross-Loop KV Compression for Looped Transformers","display_name":"Looped Latent Attention: Cross-Loop KV Compression for Looped Transformers","publication_year":2026,"publication_date":"2026-07-16","ids":{"openalex":"https://openalex.org/W7169794564","doi":"https://doi.org/10.48550/arxiv.2607.15456"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.15456","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.15456","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.15456","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5141178632","display_name":"James O' Neill","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Neill, James O'","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5074841928","display_name":"Fergal Reid","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Reid, Fergal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.545799970626831,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.545799970626831,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.04670000076293945,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.03610000014305115,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.7696999907493591},{"id":"https://openalex.org/keywords/codec","display_name":"Codec","score":0.5950999855995178},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.46709999442100525},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.43639999628067017},{"id":"https://openalex.org/keywords/path","display_name":"Path (computing)","score":0.4223000109195709},{"id":"https://openalex.org/keywords/compression-artifact","display_name":"Compression artifact","score":0.39820000529289246}],"concepts":[{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.7696999907493591},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6942999958992004},{"id":"https://openalex.org/C161765866","wikidata":"https://www.wikidata.org/wiki/Q184748","display_name":"Codec","level":2,"score":0.5950999855995178},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5656999945640564},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.46709999442100525},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.43639999628067017},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.43560001254081726},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.4223000109195709},{"id":"https://openalex.org/C57654395","wikidata":"https://www.wikidata.org/wiki/Q1097775","display_name":"Compression artifact","level":5,"score":0.39820000529289246},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.391400009393692},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.3156000077724457},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.30090001225471497},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.29679998755455017},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.2696000039577484},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.26179999113082886},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.2605000138282776}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.15456","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.15456","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.15456","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.15456","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Looped,":[0],"weight-tied":[1],"Transformers":[2],"reduce":[3],"parameters":[4],"by":[5],"reusing":[6],"a":[7,14,32,41,62,146],"single":[8,147],"block,":[9],"but":[10,141],"decoding":[11],"still":[12],"stores":[13,67],"separate":[15],"K/V":[16,38,76],"cache":[17,27,64,120,138,173],"for":[18,97,200],"every":[19],"recurrence":[20],"step.":[21],"We":[22,56],"show":[23],"that":[24,66,135],"this":[25],"loop-indexed":[26],"is":[28,103,139,175],"highly":[29],"structured.":[30],"For":[31],"fixed":[33],"token,":[34],"layer":[35,51],"and":[36,50,70,73,111,115,131,156,222],"head,":[37],"vectors":[39,77],"trace":[40],"short":[42],"low-rank":[43,140],"trajectory":[44],"across":[45],"loops,":[46],"while":[47,89],"the":[48,98,106,136,180],"head":[49],"axes":[52],"remain":[53],"much":[54],"flatter.":[55],"introduce":[57],"Looped":[58],"Latent":[59],"Attention":[60],"(\\lla{}),":[61],"post-training":[63],"codec":[65,86,102,163],"compact":[68],"K":[69],"V":[71],"latents":[72],"reconstructs":[74],"loop-specific":[75],"only":[78],"when":[79,226],"attention":[80],"reads":[81],"them.":[82],"The":[83,101,149,172],"default":[84],"per-head":[85,122],"compresses":[87],"recurrence,":[88],"\\lla{}-2D":[90],"also":[91],"folds":[92],"heads":[93],"into":[94],"one":[95,178],"latent":[96],"extreme-compression":[99],"regime.":[100],"initialized":[104],"from":[105,191,218],"SVD":[107,162],"of":[108],"teacher":[109],"activations":[110],"refined":[112],"with":[113],"logit":[114],"attention-output":[116],"distillation.":[117,231],"At":[118],"matched":[119],"budget,":[121],"\\lla{}":[123],"outperforms":[124],"head-axis":[125],"MLA,":[126],"cross-layer":[127],"sharing,":[128],"KV":[129],"quantization":[130],"final-loop":[132],"reuse,":[133],"showing":[134],"recurrent":[137],"not":[142],"safely":[143],"collapsible":[144],"to":[145,158,166,193,220,228],"state.":[148],"same":[150],"axis":[151],"advantage":[152],"holds":[153],"on":[154,210],"Ouro-2.6B-Thinking":[155],"transfers":[157],"Huginn-3.5B,":[159],"where":[160],"an":[161],"remains":[164],"near-lossless":[165],"$32\\times$":[167],"compression":[168,217],"in":[169,206],"decoder-independent":[170],"evaluation.":[171],"reduction":[174],"exact.":[176],"On":[177],"H200,":[179],"latent-store":[181],"path":[182],"increases":[183],"measured":[184],"Ouro-1.4B":[185],"batch":[186],"capacity":[187],"at":[188,196,215],"4k":[189],"context":[190],"32":[192],"768":[194],"sequences":[195],"$21.3\\times$":[197],"compression.":[198],"Lastly,":[199],"long":[201],"reasoning":[202],"rollouts":[203],"such":[204],"as":[205],"MATH-500,":[207],"on-policy":[208],"refinement":[209],"student-generated":[211],"prefixes":[212],"raises":[213],"accuracy":[214],"$4\\times$":[216],"0.43":[219],"0.66":[221],"reduces":[223],"no-answer":[224],"generations":[225],"compared":[227],"token-level":[229],"off-policy":[230]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-21T00:00:00"}