{"id":"https://openalex.org/W4401750170","doi":"https://doi.org/10.1109/jstsp.2024.3446173","title":"S$^\\text{3}$Attention: Improving Long Sequence Attention With Smoothed Skeleton Sketching","display_name":"S$^\\text{3}$Attention: Improving Long Sequence Attention With Smoothed Skeleton Sketching","publication_year":2024,"publication_date":"2024-08-22","ids":{"openalex":"https://openalex.org/W4401750170","doi":"https://doi.org/10.1109/jstsp.2024.3446173"},"language":"en","primary_location":{"id":"doi:10.1109/jstsp.2024.3446173","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2024.3446173","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2408.08567","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Xue Wang","orcid":"https://orcid.org/0009-0004-2296-9688"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]},{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Xue Wang","raw_affiliation_strings":["Alibaba Group, Bellevue, WA, USA"],"raw_orcid":"https://orcid.org/0009-0004-2296-9688","affiliations":[{"raw_affiliation_string":"Alibaba Group, Bellevue, WA, USA","institution_ids":["https://openalex.org/I4210095624","https://openalex.org/I4210108985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101890076","display_name":"Tian Zhou","orcid":"https://orcid.org/0000-0003-1789-5413"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]},{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tian Zhou","raw_affiliation_strings":["Alibaba Group, Bellevue, WA, USA"],"raw_orcid":"https://orcid.org/0000-0003-1789-5413","affiliations":[{"raw_affiliation_string":"Alibaba Group, Bellevue, WA, USA","institution_ids":["https://openalex.org/I4210095624","https://openalex.org/I4210108985"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Jianqing Zhu","orcid":"https://orcid.org/0000-0003-4563-6785"},"institutions":[{"id":"https://openalex.org/I71920554","display_name":"King Abdullah University of Science and Technology","ror":"https://ror.org/01q3tbs38","country_code":"SA","type":"education","lineage":["https://openalex.org/I71920554"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Jianqing Zhu","raw_affiliation_strings":["Computer, Electrical and Mathematical Science and Engineering Division, King Abdullah University of Science and Technology, Thuwal, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0003-4563-6785","affiliations":[{"raw_affiliation_string":"Computer, Electrical and Mathematical Science and Engineering Division, King Abdullah University of Science and Technology, Thuwal, Saudi Arabia","institution_ids":["https://openalex.org/I71920554"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100456648","display_name":"Jialin Liu","orcid":"https://orcid.org/0000-0003-0291-3201"},"institutions":[{"id":"https://openalex.org/I106165777","display_name":"University of Central Florida","ror":"https://ror.org/036nfer12","country_code":"US","type":"education","lineage":["https://openalex.org/I106165777"]},{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]},{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jialin Liu","raw_affiliation_strings":["Department of Statistics and Data Science, University of Central Florida, Orlando, FL, USA","Alibaba Group, Bellevue, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Statistics and Data Science, University of Central Florida, Orlando, FL, USA","institution_ids":["https://openalex.org/I106165777"]},{"raw_affiliation_string":"Alibaba Group, Bellevue, WA, USA","institution_ids":["https://openalex.org/I4210095624","https://openalex.org/I4210108985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100614598","display_name":"Kun Yuan","orcid":"https://orcid.org/0000-0001-8394-8187"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kun Yuan","raw_affiliation_strings":["Center for Machine Learning Research, Peking University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Machine Learning Research, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009869768","display_name":"Tao Yao","orcid":"https://orcid.org/0000-0003-0241-2044"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tao Yao","raw_affiliation_strings":["Antai College of Economics and Management, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Antai College of Economics and Management, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085908411","display_name":"Wotao Yin","orcid":"https://orcid.org/0000-0001-6697-9731"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]},{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wotao Yin","raw_affiliation_strings":["Alibaba Group, Bellevue, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Group, Bellevue, WA, USA","institution_ids":["https://openalex.org/I4210095624","https://openalex.org/I4210108985"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Rong Jin","orcid":"https://orcid.org/0000-0002-8797-4646"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rong Jin","raw_affiliation_strings":["Meta, Menlo Park, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-8797-4646","affiliations":[{"raw_affiliation_string":"Meta, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210099336"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103070466","display_name":"HanQin Cai","orcid":"https://orcid.org/0000-0002-2937-1986"},"institutions":[{"id":"https://openalex.org/I106165777","display_name":"University of Central Florida","ror":"https://ror.org/036nfer12","country_code":"US","type":"education","lineage":["https://openalex.org/I106165777"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"HanQin Cai","raw_affiliation_strings":["Department of Statistics and Data Science and the Department of Computer Science, University of Central Florida, Orlando, FL, USA","Department of Statistics and Data Science and Department of Computer Science, University of Central Florida, Orlando, FL, USA"],"raw_orcid":"https://orcid.org/0000-0002-2937-1986","affiliations":[{"raw_affiliation_string":"Department of Statistics and Data Science and the Department of Computer Science, University of Central Florida, Orlando, FL, USA","institution_ids":["https://openalex.org/I106165777"]},{"raw_affiliation_string":"Department of Statistics and Data Science and Department of Computer Science, University of Central Florida, Orlando, FL, USA","institution_ids":["https://openalex.org/I106165777"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":9,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210095624","https://openalex.org/I4210108985"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.1119215,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"18","issue":"6","first_page":"985","last_page":"996"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.8992999792098999,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.8992999792098999,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.6457052230834961},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5758756995201111},{"id":"https://openalex.org/keywords/skeleton","display_name":"Skeleton (computer programming)","score":0.5177949666976929},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3902677297592163},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3811253309249878},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.372905433177948},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.12102660536766052}],"concepts":[{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.6457052230834961},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5758756995201111},{"id":"https://openalex.org/C18969341","wikidata":"https://www.wikidata.org/wiki/Q1169129","display_name":"Skeleton (computer programming)","level":2,"score":0.5177949666976929},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3902677297592163},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3811253309249878},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.372905433177948},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.12102660536766052},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/jstsp.2024.3446173","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2024.3446173","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2408.08567","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.08567","pdf_url":"https://arxiv.org/pdf/2408.08567","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2408.08567","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.08567","pdf_url":"https://arxiv.org/pdf/2408.08567","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4705590573","display_name":null,"funder_award_id":"DMS 2304489","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5446884816","display_name":null,"funder_award_id":"2304489","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4401750170.pdf"},"referenced_works_count":81,"referenced_works":["https://openalex.org/W1998269045","https://openalex.org/W2152402969","https://openalex.org/W2611328865","https://openalex.org/W2753069234","https://openalex.org/W2795632874","https://openalex.org/W2896457183","https://openalex.org/W2948429466","https://openalex.org/W2964110616","https://openalex.org/W2964321615","https://openalex.org/W3012775393","https://openalex.org/W3015286511","https://openalex.org/W3022488072","https://openalex.org/W3022749512","https://openalex.org/W3085139254","https://openalex.org/W3093365368","https://openalex.org/W3094502228","https://openalex.org/W3106298483","https://openalex.org/W3115071167","https://openalex.org/W3131922516","https://openalex.org/W3134139903","https://openalex.org/W3138516171","https://openalex.org/W3155157989","https://openalex.org/W3162090017","https://openalex.org/W3173539742","https://openalex.org/W3175710487","https://openalex.org/W3177318507","https://openalex.org/W3196660197","https://openalex.org/W3205245241","https://openalex.org/W4224309822","https://openalex.org/W4295308583","https://openalex.org/W4360752541","https://openalex.org/W4385245566","https://openalex.org/W4385346076","https://openalex.org/W4389519981","https://openalex.org/W4391224793","https://openalex.org/W4393407053","https://openalex.org/W6748148878","https://openalex.org/W6754454322","https://openalex.org/W6761628794","https://openalex.org/W6763509872","https://openalex.org/W6764679822","https://openalex.org/W6766673545","https://openalex.org/W6767368257","https://openalex.org/W6770251742","https://openalex.org/W6771626834","https://openalex.org/W6771917389","https://openalex.org/W6774418012","https://openalex.org/W6776048684","https://openalex.org/W6776684981","https://openalex.org/W6778883912","https://openalex.org/W6779089016","https://openalex.org/W6779163297","https://openalex.org/W6779709467","https://openalex.org/W6781533629","https://openalex.org/W6782420349","https://openalex.org/W6783944145","https://openalex.org/W6785783668","https://openalex.org/W6788135285","https://openalex.org/W6789317445","https://openalex.org/W6791838475","https://openalex.org/W6796753453","https://openalex.org/W6797019122","https://openalex.org/W6797155008","https://openalex.org/W6798335185","https://openalex.org/W6799488796","https://openalex.org/W6799838802","https://openalex.org/W6801910562","https://openalex.org/W6802868300","https://openalex.org/W6802912272","https://openalex.org/W6803444062","https://openalex.org/W6803916128","https://openalex.org/W6804239712","https://openalex.org/W6809915218","https://openalex.org/W6810370753","https://openalex.org/W6810615099","https://openalex.org/W6810637551","https://openalex.org/W6839228074","https://openalex.org/W6839395979","https://openalex.org/W6841205203","https://openalex.org/W6843425716","https://openalex.org/W6843735039"],"related_works":["https://openalex.org/W2772917594","https://openalex.org/W2036807459","https://openalex.org/W2058170566","https://openalex.org/W2755342338","https://openalex.org/W2166024367","https://openalex.org/W3116076068","https://openalex.org/W2229312674","https://openalex.org/W2951359407","https://openalex.org/W2079911747","https://openalex.org/W1969923398"],"abstract_inverted_index":{"Attention":[0,16,20,32,100,200,206],"based":[1,21,99],"models":[2,22],"have":[3],"achieved":[4],"many":[5],"remarkable":[6],"breakthroughs":[7],"in":[8],"numerous":[9],"applications.":[10],"However,":[11],"the":[12,18,38,47,61,70,74,81,111,128,134,138,162,167],"quadratic":[13],"complexity":[14,136],"of":[15,56,83,130,169,205],"makes":[17],"vanilla":[19,199],"hard":[23],"to":[24,26,36,114,125,137,144],"apply":[25],"long":[27,148],"sequence":[28,49,139],"tasks.":[29],"Various":[30],"improved":[31],"structures":[33],"are":[34],"proposed":[35],"reduce":[37],"computation":[39,68],"cost":[40],"by":[41,50],"inducing":[42],"low":[43],"rankness":[44],"and":[45,67,87,150,159,176,186,201],"approximating":[46],"whole":[48],"sub-sequences.":[51],"The":[52],"most":[53],"challenging":[54],"part":[55],"those":[57],"approaches":[58],"is":[59,77],"maintaining":[60],"proper":[62],"balance":[63],"between":[64],"information":[65,76,146],"preservation":[66],"reduction:":[69],"longer":[71],"sub-sequences":[72],"used,":[73],"better":[75],"preserved,":[78],"but":[79],"at":[80],"price":[82],"introducing":[84],"more":[85],"noise":[86,131],"computational":[88],"costs.":[89],"In":[90],"this":[91,116],"paper,":[92],"we":[93],"propose":[94],"a":[95,141,151],"smoothed":[96],"skeleton":[97],"sketching":[98,153],"structure,":[101],"coined":[102],"S<inline-formula":[103,118,170,192],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[104,119,171,193],"xmlns:xlink=\"http://www.w3.org/1999/xlink\"><tex-math":[105,120,172,194],"notation=\"LaTeX\">$^{3}$</tex-math></inline-formula>Attention,":[106],"which":[107],"significantly":[108,196],"improves":[109],"upon":[110],"previous":[112],"attempts":[113],"negotiate":[115],"trade-off.":[117],"notation=\"LaTeX\">$^{3}$</tex-math></inline-formula>Attention":[121,173,195],"has":[122],"two":[123],"mechanisms":[124],"effectively":[126],"minimize":[127],"impact":[129],"while":[132],"keeping":[133],"linear":[135],"length:":[140],"smoothing":[142],"block":[143],"mix":[145],"over":[147,180],"sequences":[149],"matrix":[152],"method":[154],"that":[155,191],"simultaneously":[156],"selects":[157],"columns":[158],"rows":[160],"from":[161],"input":[163],"matrix.":[164],"We":[165],"verify":[166],"effectiveness":[168],"both":[174,198],"theoretically":[175],"empirically.":[177],"Extensive":[178],"studies":[179],"Long":[181],"Range":[182],"Arena":[183],"(LRA)":[184],"datasets":[185],"six":[187],"time-series":[188],"forecasting":[189],"show":[190],"outperforms":[197],"other":[202],"state-of-the-art":[203],"variants":[204],"structures.":[207]},"counts_by_year":[],"updated_date":"2025-12-21T23:12:01.093139","created_date":"2024-08-23T00:00:00"}