{"id":"https://openalex.org/W7127339647","doi":"https://doi.org/10.48550/arxiv.2602.01572","title":"LLM-based Embeddings: Attention Values Encode Sentence Semantics Better Than Hidden States","display_name":"LLM-based Embeddings: Attention Values Encode Sentence Semantics Better Than Hidden States","publication_year":2026,"publication_date":"2026-02-02","ids":{"openalex":"https://openalex.org/W7127339647","doi":"https://doi.org/10.48550/arxiv.2602.01572"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.01572","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5124937421","display_name":"Yeqin Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Zhang, Yeqin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124874956","display_name":"Yunfei Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yunfei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124963631","display_name":"Jiaxuan Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jiaxuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124890936","display_name":"Ke Qin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qin, Ke","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124932253","display_name":"Yizheng Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yizheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5065000009","display_name":"Cam-Tu Nguyen","orcid":"https://orcid.org/0009-0006-9484-6876"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Cam-Tu","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5124937421"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4138999879360199,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4138999879360199,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.11569999903440475,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.06430000066757202,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.6940000057220459},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6686000227928162},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.6664999723434448},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.5223000049591064},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.4805000126361847},{"id":"https://openalex.org/keywords/encode","display_name":"ENCODE","score":0.4706999957561493},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.37860000133514404},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.36820000410079956}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7148000001907349},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.6940000057220459},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6686000227928162},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6664999723434448},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5278000235557556},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.5223000049591064},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.4805000126361847},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.4706999957561493},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4571000039577484},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.37860000133514404},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.36820000410079956},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.36629998683929443},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.33570000529289246},{"id":"https://openalex.org/C2778828372","wikidata":"https://www.wikidata.org/wiki/Q5283209","display_name":"Distributional semantics","level":3,"score":0.32330000400543213},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.32089999318122864},{"id":"https://openalex.org/C57493831","wikidata":"https://www.wikidata.org/wiki/Q3134666","display_name":"Projection (relational algebra)","level":2,"score":0.30570000410079956},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.3019999861717224},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.29260000586509705},{"id":"https://openalex.org/C2777462759","wikidata":"https://www.wikidata.org/wiki/Q18395344","display_name":"Word embedding","level":3,"score":0.28870001435279846},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.2685999870300293},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.2646999955177307},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.2644999921321869},{"id":"https://openalex.org/C2780069185","wikidata":"https://www.wikidata.org/wiki/Q7977945","display_name":"Equivalence (formal languages)","level":2,"score":0.2630999982357025},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.2517000138759613}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.01572","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.01572","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.01572","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.01572","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.6742863655090332}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Sentence":[0],"representations":[1],"are":[2,30],"foundational":[3],"to":[4,19,39],"many":[5],"Natural":[6],"Language":[7,16],"Processing":[8],"(NLP)":[9],"applications.":[10],"While":[11],"recent":[12],"methods":[13],"leverage":[14],"Large":[15],"Models":[17],"(LLMs)":[18],"derive":[20],"sentence":[21,56],"representations,":[22],"most":[23],"rely":[24],"on":[25],"final-layer":[26],"hidden":[27,61],"states,":[28],"which":[29],"optimized":[31],"for":[32],"next-token":[33],"prediction":[34],"and":[35,78],"thus":[36],"often":[37],"fail":[38],"capture":[40,55],"global,":[41],"sentence-level":[42],"semantics.":[43],"This":[44,150],"paper":[45],"introduces":[46],"a":[47,68,82,170],"novel":[48],"perspective,":[49],"demonstrating":[50],"that":[51,71,100],"attention":[52,108,120],"value":[53,116,139],"vectors":[54,140],"semantics":[57],"more":[58],"effectively":[59],"than":[60],"states.":[62],"We":[63],"propose":[64],"Value":[65,186],"Aggregation":[66],"(VA),":[67],"simple":[69],"method":[70],"pools":[72],"token":[73,79,125],"values":[74],"across":[75],"multiple":[76],"layers":[77],"indices.":[80],"In":[81],"training-free":[83,162],"setting,":[84],"VA":[85,156],"outperforms":[86],"other":[87],"LLM-based":[88,163],"embeddings,":[89,164],"even":[90],"matches":[91],"or":[92],"surpasses":[93],"the":[94,106,119,123,128,131,142,146,166,176],"ensemble-based":[95],"MetaEOL.":[96],"Furthermore,":[97],"we":[98,174],"demonstrate":[99],"when":[101],"paired":[102],"with":[103,141],"suitable":[104],"prompts,":[105],"layer":[107],"outputs":[109],"can":[110],"be":[111],"interpreted":[112],"as":[113,127],"aligned":[114],"weighted":[115,138],"vectors.":[117],"Specifically,":[118],"scores":[121],"of":[122,145,178],"last":[124],"function":[126],"weights,":[129],"while":[130],"output":[132],"projection":[133],"matrix":[134],"($W_O$)":[135],"aligns":[136],"these":[137],"common":[143],"space":[144],"LLM":[147,181],"residual":[148],"stream.":[149],"refined":[151],"method,":[152],"termed":[153],"Aligned":[154],"Weighted":[155],"(AlignedWVA),":[157],"achieves":[158],"state-of-the-art":[159],"performance":[160],"among":[161],"outperforming":[165],"high-cost":[167],"MetaEOL":[168],"by":[169],"substantial":[171],"margin.":[172],"Finally,":[173],"highlight":[175],"potential":[177],"obtaining":[179],"strong":[180],"embedding":[182],"models":[183],"through":[184],"fine-tuning":[185],"Aggregation.":[187]},"counts_by_year":[],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2026-02-04T00:00:00"}