{"id":"https://openalex.org/W4405857437","doi":"https://doi.org/10.1145/3705956.3705968","title":"Enhancing HyperAttention: A Novel Approach for Improved Algorithmic Efficiency","display_name":"Enhancing HyperAttention: A Novel Approach for Improved Algorithmic Efficiency","publication_year":2024,"publication_date":"2024-07-05","ids":{"openalex":"https://openalex.org/W4405857437","doi":"https://doi.org/10.1145/3705956.3705968"},"language":"en","primary_location":{"id":"doi:10.1145/3705956.3705968","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3705956.3705968","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 8th High Performance Computing and Cluster Technologies Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3705956.3705968","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Xinyi Xie","orcid":"https://orcid.org/0009-0000-1259-756X"},"institutions":[{"id":"https://openalex.org/I12615008","display_name":"Beijing Normal-Hong Kong Baptist University","ror":"https://ror.org/04snvc712","country_code":"CN","type":"education","lineage":["https://openalex.org/I12615008"]},{"id":"https://openalex.org/I141568987","display_name":"Hong Kong Baptist University","ror":"https://ror.org/0145fw131","country_code":"HK","type":"education","lineage":["https://openalex.org/I141568987"]},{"id":"https://openalex.org/I25254941","display_name":"Beijing Normal University","ror":"https://ror.org/022k4wk35","country_code":"CN","type":"education","lineage":["https://openalex.org/I25254941"]}],"countries":["CN","HK"],"is_corresponding":true,"raw_author_name":"Xinyi Xie","raw_affiliation_strings":["Faculty of Science and Technology, Beijing Normal University-Hong Kong Baptist University United International College, Zhuhai, China"],"raw_orcid":"https://orcid.org/0009-0000-1259-756X","affiliations":[{"raw_affiliation_string":"Faculty of Science and Technology, Beijing Normal University-Hong Kong Baptist University United International College, Zhuhai, China","institution_ids":["https://openalex.org/I12615008","https://openalex.org/I25254941","https://openalex.org/I141568987"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016520313","display_name":"Yi Ding","orcid":null},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yi Ding","raw_affiliation_strings":["School of Information Science and Technology, Xiamen University Tan Kan Kee College, Zhangzhou, China"],"raw_orcid":"https://orcid.org/0009-0004-4383-6374","affiliations":[{"raw_affiliation_string":"School of Information Science and Technology, Xiamen University Tan Kan Kee College, Zhangzhou, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115682560","display_name":"Chao Jiang","orcid":null},"institutions":[{"id":"https://openalex.org/I2799850029","display_name":"Dongguan University of Technology","ror":"https://ror.org/01m8p7q42","country_code":"CN","type":"education","lineage":["https://openalex.org/I2799850029"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chao Jiang","raw_affiliation_strings":["School of Computer Science and Technology, Dongguan University of Technology, Dongguan, China"],"raw_orcid":"https://orcid.org/0009-0002-3426-8219","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Dongguan University of Technology, Dongguan, China","institution_ids":["https://openalex.org/I2799850029"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115682561","display_name":"Tianyi Hang","orcid":null},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianyi Hang","raw_affiliation_strings":["School of Information Science and Technology, Xi'an Jiaotong University, Xi'an, China"],"raw_orcid":"https://orcid.org/0009-0006-1109-5975","affiliations":[{"raw_affiliation_string":"School of Information Science and Technology, Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5105525881","display_name":"Tan Lu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210166603","display_name":"Jinling Institute of Technology","ror":"https://ror.org/05em1gq62","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210166603"]},{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianrun Lu","raw_affiliation_strings":["Jinling College, Nanjing University, Heibei, China"],"raw_orcid":"https://orcid.org/0009-0006-9672-9307","affiliations":[{"raw_affiliation_string":"Jinling College, Nanjing University, Heibei, China","institution_ids":["https://openalex.org/I4210166603","https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I12615008","https://openalex.org/I141568987","https://openalex.org/I25254941"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.23581543,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"18","last_page":"23"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7019144296646118}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7019144296646118}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3705956.3705968","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3705956.3705968","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 8th High Performance Computing and Cluster Technologies Conference","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3705956.3705968","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3705956.3705968","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 8th High Performance Computing and Cluster Technologies Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/7","score":0.41999998688697815,"display_name":"Affordable and clean energy"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W1977556410","https://openalex.org/W2127218421","https://openalex.org/W2147717514","https://openalex.org/W4236362309","https://openalex.org/W4281758439","https://openalex.org/W4387595597","https://openalex.org/W6600047755","https://openalex.org/W6629956336","https://openalex.org/W6739901393","https://openalex.org/W6781533629"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"HyperAttention":[0,50,84,114,158],"is":[1,104,201],"a":[2,43,65,70,178,209],"new":[3],"attention":[4],"mechanism":[5],"proposed":[6],"to":[7,30,74,78,97,105,149,192,226,238],"address":[8],"the":[9,16,28,32,48,52,59,81,94,107,127,153,174,196,204,220,231,235,240],"inevitability":[10],"of":[11,18,54,58,83,155,157,161,222],"quadratic":[12],"time":[13,20,37,183,199],"complexity":[14,34,200],"and":[15,56,69,119,137,164,180,214,229],"conditionality":[17],"linear":[19,36],"when":[21],"dealing":[22],"with":[23,63,190],"extended":[24],"contexts,":[25],"which":[26,143],"allows":[27],"algorithm":[29,96,111,176],"maintain":[31],"computational":[33],"in":[35,47,113,147,159,198,206],"despite":[38],"having":[39],"large":[40,44,60,117],"entries":[41],"or":[42],"stability":[45],"rank":[46],"matrix.":[49],"improves":[51],"speed":[53],"inference":[55],"training":[57],"language":[61],"model,":[62],"only":[64],"slight":[66],"performance":[67,213],"degradation":[68],"significant":[71],"speedup":[72],"compared":[73],"FlashAttention.":[75],"In":[76,130],"order":[77],"further":[79,125],"improve":[80,98,227],"accuracy":[82,228],"while":[85],"maintaining":[86],"its":[87,99],"speed,":[88],"we":[89,132,217],"optimised":[90],"each":[91],"module":[92],"for":[93,115,123,152,233],"original":[95],"efficiency.":[100,215],"Our":[101],"main":[102],"contribution":[103],"change":[106],"locally":[108],"sensitive":[109],"hash":[110],"used":[112],"detecting":[116],"values":[118],"instead":[120],"use":[121],"k-means":[122],"indexing,":[124],"reducing":[126],"model's":[128],"perplexity.":[129,188],"addition,":[131],"add":[133],"applications":[134],"on":[135],"chatglm3-6b":[136],"baichuan-7b":[138],"as":[139,141],"well":[140],"chatglm3-6b-32k":[142],"supports":[144],"long":[145],"contexts":[146],"practice":[148],"make":[150],"up":[151],"lack":[154],"informativeness":[156],"terms":[160],"practical":[162],"applications,":[163],"test":[165],"results":[166],"are":[167],"given.":[168],"The":[169],"experiments":[170],"reveal":[171],"that":[172],"although":[173],"improved":[175],"has":[177],"longer":[179],"relatively":[181],"stable":[182],"complexity,":[184],"it":[185],"significantly":[186],"reduces":[187],"Notably,":[189],"10":[191],"15":[193],"replacement":[194],"layers,":[195],"increase":[197],"less":[202],"than":[203],"reduction":[205],"perplexity,":[207],"achieving":[208],"favourable":[210],"balance":[211],"between":[212],"Moreover,":[216],"also":[218],"analyse":[219],"feasibility":[221],"other":[223],"sampling":[224],"methods":[225],"discuss":[230],"strategy":[232],"selecting":[234],"patch":[236],"layers":[237],"apply":[239],"fast":[241],"algorithm.":[242]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}