{"id":"https://openalex.org/W7160391455","doi":"https://doi.org/10.1162/tacl.a.616","title":"Cross-layer Attention Sharing for Pre-trained Large Language Models","display_name":"Cross-layer Attention Sharing for Pre-trained Large Language Models","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7160391455","doi":"https://doi.org/10.1162/tacl.a.616"},"language":"en","primary_location":{"id":"doi:10.1162/tacl.a.616","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl.a.616","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1162/tacl.a.616","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024348709","display_name":"Yongyu Mu","orcid":null},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongyu Mu","raw_affiliation_strings":["NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China. lixiaoyumu9@gmail.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China. lixiaoyumu9@gmail.com","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067915781","display_name":"Yuzhang Wu","orcid":"https://orcid.org/0000-0002-4049-0214"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuzhang Wu","raw_affiliation_strings":["NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102121415","display_name":"Yuchun Fan","orcid":null},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuchun Fan","raw_affiliation_strings":["NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059868742","display_name":"C Q Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chenglong Wang","raw_affiliation_strings":["NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021861548","display_name":"Hengyu Li","orcid":null},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hengyu Li","raw_affiliation_strings":["NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135472588","display_name":"Jiali Zeng","orcid":null},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiali Zeng","raw_affiliation_strings":["Pattern Recognition Center, WeChat AI, Tencent Inc, China. lemonzeng@tencent.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pattern Recognition Center, WeChat AI, Tencent Inc, China. lemonzeng@tencent.com","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135477892","display_name":"Qiaozhi He","orcid":null},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qiaozhi He","raw_affiliation_strings":["NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101207490","display_name":"Murun Yang","orcid":"https://orcid.org/0000-0002-4468-9706"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Murun Yang","raw_affiliation_strings":["NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135510930","display_name":"Fandong Meng","orcid":null},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fandong Meng","raw_affiliation_strings":["Pattern Recognition Center, WeChat AI, Tencent Inc, China. fandongmeng@tencent.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pattern Recognition Center, WeChat AI, Tencent Inc, China. fandongmeng@tencent.com","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135432757","display_name":"Jie Zhou","orcid":null},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jie Zhou","raw_affiliation_strings":["Pattern Recognition Center, WeChat AI, Tencent Inc, China. withtomzhou@tencent.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pattern Recognition Center, WeChat AI, Tencent Inc, China. withtomzhou@tencent.com","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135423543","display_name":"Tong Xiao","orcid":null},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tong Xiao","raw_affiliation_strings":["NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China. xiaotong@mail.neu.edu.cn"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China. xiaotong@mail.neu.edu.cn","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5135514774","display_name":"Jingbo Zhu","orcid":null},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingbo Zhu","raw_affiliation_strings":["NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China. zhujingbo@mail.neu.edu.cn"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Lab, School of Computer Science and Engineering, Northeastern University, Shenyang, China. zhujingbo@mail.neu.edu.cn","institution_ids":["https://openalex.org/I9224756"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.58846608,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"14","issue":null,"first_page":"656","last_page":"688"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.14959999918937683,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.14959999918937683,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.10970000177621841,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.09260000288486481,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.867900013923645},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.6876000165939331},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.5791000127792358},{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.45660001039505005},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4555000066757202},{"id":"https://openalex.org/keywords/attention-network","display_name":"Attention network","score":0.42080000042915344}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8824999928474426},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.867900013923645},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.6876000165939331},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.5791000127792358},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.45660001039505005},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4555000066757202},{"id":"https://openalex.org/C2993807640","wikidata":"https://www.wikidata.org/wiki/Q103709453","display_name":"Attention network","level":2,"score":0.42080000042915344},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3562999963760376},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3458999991416931},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3456000089645386},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33230000734329224},{"id":"https://openalex.org/C67820243","wikidata":"https://www.wikidata.org/wiki/Q179164","display_name":"Unitary state","level":2,"score":0.30709999799728394},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.29649999737739563},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.28029999136924744},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27000001072883606},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.26109999418258667},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.26080000400543213}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1162/tacl.a.616","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl.a.616","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1162/tacl.a.616","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl.a.616","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W1902237438","https://openalex.org/W2887783173","https://openalex.org/W2890894339","https://openalex.org/W2912924812","https://openalex.org/W2946609015","https://openalex.org/W2946794439","https://openalex.org/W2962677625","https://openalex.org/W2963015836","https://openalex.org/W2963339397","https://openalex.org/W2964089206","https://openalex.org/W2964213727","https://openalex.org/W2964223283","https://openalex.org/W2965046076","https://openalex.org/W2998183051","https://openalex.org/W2998617917","https://openalex.org/W3017024317","https://openalex.org/W3034457371","https://openalex.org/W3035251378","https://openalex.org/W3038012435","https://openalex.org/W3085139254","https://openalex.org/W3088982929","https://openalex.org/W3095173472","https://openalex.org/W3105238007","https://openalex.org/W3105966348","https://openalex.org/W3115511229","https://openalex.org/W3131922516","https://openalex.org/W3134255974","https://openalex.org/W3167266074","https://openalex.org/W3169597853","https://openalex.org/W3194676777","https://openalex.org/W3199348444","https://openalex.org/W4229005866","https://openalex.org/W4249736682","https://openalex.org/W4291653336","https://openalex.org/W4308760226","https://openalex.org/W4322718191","https://openalex.org/W4384918448","https://openalex.org/W4387929589","https://openalex.org/W4389217698","https://openalex.org/W4389518760","https://openalex.org/W4389524067","https://openalex.org/W4392678657","https://openalex.org/W4392822465","https://openalex.org/W4396815229","https://openalex.org/W4399414358","https://openalex.org/W4402667093","https://openalex.org/W4402670094","https://openalex.org/W4402670433","https://openalex.org/W4402684173","https://openalex.org/W4404689873","https://openalex.org/W4404782141","https://openalex.org/W4404782803","https://openalex.org/W4412887764","https://openalex.org/W4414359404","https://openalex.org/W4415796681","https://openalex.org/W4416413428","https://openalex.org/W7133227460"],"related_works":[],"abstract_inverted_index":{"Abstract":[0],"To":[1],"enhance":[2],"the":[3,6,18,51,68,74,156,174],"efficiency":[4],"of":[5,143,155,161,168],"attention":[7,23,41,55,75,90,114,127,150,175],"mechanism":[8],"within":[9,44,152,173],"large":[10],"language":[11],"models":[12],"(LLMs),":[13],"previous":[14],"works":[15],"primarily":[16],"compress":[17],"Key-Value":[19],"cache":[20],"or":[21],"group":[22],"heads,":[24],"while":[25,147],"largely":[26],"overlooking":[27],"redundancy":[28,52],"between":[29,116],"layers.":[30,46,58,158],"Our":[31,159,191],"comprehensive":[32],"analyses":[33],"across":[34,57],"various":[35],"LLMs":[36],"show":[37],"that":[38,135],"highly":[39],"similar":[40],"patterns":[42],"persist":[43],"most":[45],"It\u2019s":[47],"intuitive":[48],"to":[49,78,86,112,122],"reduce":[50],"by":[53,93],"sharing":[54,67],"weights":[56],"However,":[59],"further":[60],"analysis":[61],"reveals":[62],"two":[63],"challenges:":[64],"(1)":[65],"Directly":[66],"weight":[69],"matrix":[70],"without":[71],"carefully":[72],"rearranging":[73],"heads":[76,115],"proves":[77],"be":[79],"ineffective;":[80],"(2)":[81],"Shallow":[82],"layers":[83,118],"are":[84],"vulnerable":[85],"small":[87],"deviations":[88],"in":[89,104,125,141],"weights.":[91,128],"Driven":[92],"these":[94],"insights,":[95],"we":[96],"introduce":[97],"LiSA,":[98],"a":[99,164],"lightweight":[100],"substitute":[101],"for":[102,185],"self-attention":[103],"well-trained":[105],"LLMs.":[106],"LiSA":[107,136,162],"employs":[108],"tiny":[109],"feed-forward":[110],"networks":[111],"align":[113],"adjacent":[117],"and":[119,145,170,183,188],"low-rank":[120],"matrices":[121,172],"approximate":[123],"differences":[124],"layer-wise":[126],"Evaluations":[129],"encompassing":[130],"13":[131],"typical":[132],"benchmarks":[133],"demonstrate":[134],"maintains":[137],"high":[138],"response":[139],"quality":[140],"terms":[142],"accuracy":[144],"perplexity":[146],"reducing":[148],"redundant":[149],"calculations":[151],"53%":[153],"\u221284%":[154],"total":[157],"implementations":[160],"achieve":[163],"6":[165],"\u00d7":[166],"compression":[167],"Q":[169],"K":[171],"mechanism,":[176],"with":[177],"maximum":[178],"throughput":[179],"improvements":[180],"19.5%,":[181],"32.3%,":[182],"40.1%":[184],"LLaMA3-8B,":[186],"LLaMA2-7B,":[187],"LLaMA2-13B,":[189],"respectively.":[190],"code":[192],"is":[193],"available":[194],"at":[195],"https://github.com/takagi97/lisa.":[196]},"counts_by_year":[],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2026-05-07T00:00:00"}