{"id":"https://openalex.org/W7163428704","doi":"https://doi.org/10.48550/arxiv.2606.02780","title":"Do Value Vectors in Deep Layers Need Context from the Residual Stream?","display_name":"Do Value Vectors in Deep Layers Need Context from the Residual Stream?","publication_year":2026,"publication_date":"2026-06-01","ids":{"openalex":"https://openalex.org/W7163428704","doi":"https://doi.org/10.48550/arxiv.2606.02780"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.02780","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.02780","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.02780","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137752423","display_name":"Muyu He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Muyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137802733","display_name":"Yuchen Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yuchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086363752","display_name":"Qingya Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Qingya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137737982","display_name":"Li Zhang (8200)","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11338","display_name":"Advancements in Photolithography Techniques","score":0.1404999941587448,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11338","display_name":"Advancements in Photolithography Techniques","score":0.1404999941587448,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.09749999642372131,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.07569999992847443,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.7202000021934509},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.5479999780654907},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.5214999914169312},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5078999996185303},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.46399998664855957},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.42329999804496765},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.37059998512268066},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.35839998722076416}],"concepts":[{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.7202000021934509},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5974000096321106},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5479999780654907},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.5214999914169312},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5078999996185303},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.46399998664855957},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4424000084400177},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.42329999804496765},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4163999855518341},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.37059998512268066},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.35839998722076416},{"id":"https://openalex.org/C22789450","wikidata":"https://www.wikidata.org/wiki/Q420904","display_name":"Singular value decomposition","level":2,"score":0.35580000281333923},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.3553999960422516},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35030001401901245},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3409000039100647},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.302700012922287},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.2976999878883362},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.2946000099182129},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.2863999903202057},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.28610000014305115},{"id":"https://openalex.org/C45235069","wikidata":"https://www.wikidata.org/wiki/Q278425","display_name":"Table (database)","level":2,"score":0.28380000591278076},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.2831000089645386},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.2721000015735626},{"id":"https://openalex.org/C193415008","wikidata":"https://www.wikidata.org/wiki/Q639681","display_name":"Network architecture","level":2,"score":0.2526000142097473}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.02780","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.02780","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.02780","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.02780","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"success":[1],"of":[2,9,20,140,146,157,163,167],"the":[3,7,27,34,67,77,81,92,115,128,164,184,191,200],"transformer":[4],"architecture":[5],"as":[6,37,110],"backbone":[8],"modern":[10],"LLMs":[11],"is":[12],"in":[13,150],"large":[14],"part":[15],"due":[16],"to":[17,65,85,117,199],"its":[18],"use":[19],"attention":[21,24,151,180],"layers.":[22,168],"An":[23],"layer":[25],"follows":[26],"standard":[28,179],"neural":[29],"network":[30],"paradigm:":[31],"it":[32],"takes":[33],"residual":[35,78],"stream":[36],"input":[38],"and":[39,45,171,206],"thereby":[40],"produces":[41],"context-dependent":[42,93],"query,":[43],"key,":[44],"value":[46,63,88,105,135,148,159,201],"vectors.":[47],"However,":[48],"we":[49,137],"find":[50],"that":[51,195],"model":[52,82,112],"performance":[53],"meaningfully":[54],"improves":[55,175],"when":[56],"deeper":[57],"layers":[58],"learn":[59],"only":[60],"a":[61,143,154],"context-free":[62,87,104,134],"vector":[64,202],"preserve":[66],"original":[68],"token":[69,197],"information,":[70],"without":[71],"drawing":[72],"on":[73,127],"any":[74],"context":[75],"from":[76],"stream.":[79],"When":[80],"has":[83],"access":[84],"this":[86],"vector,":[89],"adding":[90],"back":[91],"component":[94],"provides":[95],"little":[96],"additional":[97],"benefit":[98],"for":[99,132,161],"aggregate":[100],"benchmark":[101],"performance.":[102],"Such":[103],"vectors":[106,149,160],"can":[107],"be":[108],"stored":[109],"sparse":[111],"parameters,":[113],"eliminating":[114],"need":[116],"recompute":[118],"or":[119],"persistently":[120],"cache":[121],"these":[122],"values.":[123],"Through":[124],"systematic":[125],"ablations":[126],"key":[129],"design":[130],"choices":[131],"such":[133],"vectors,":[136],"propose":[138],"Bank":[139],"Values":[141],"(BoV),":[142],"new":[144],"way":[145],"computing":[147],"by":[152],"learning":[153],"lookup":[155],"table":[156],"token-specific":[158],"each":[162],"last":[165],"third":[166],"Across":[169],"135M":[170],"780M":[172],"models,":[173],"BoV":[174],"validation":[176],"loss":[177],"over":[178],"and,":[181],"at":[182],"780M,":[183],"average":[185],"score":[186],"across":[187],"21":[188],"benchmarks,":[189],"matching":[190],"previous":[192],"best":[193],"method":[194],"adds":[196],"information":[198],"with":[203],"less":[204],"compute":[205],"memory.":[207]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-04T00:00:00"}