{"id":"https://openalex.org/W7117538323","doi":"https://doi.org/10.1109/access.2025.3649490","title":"Learnable Temporal Sparse Memory iTransformer: Revisiting Sparsity and Memory in Transformers","display_name":"Learnable Temporal Sparse Memory iTransformer: Revisiting Sparsity and Memory in Transformers","publication_year":2025,"publication_date":"2025-12-30","ids":{"openalex":"https://openalex.org/W7117538323","doi":"https://doi.org/10.1109/access.2025.3649490"},"language":null,"primary_location":{"id":"doi:10.1109/access.2025.3649490","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3649490","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/access.2025.3649490","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050895436","display_name":"Polycarp Shizawaliyi Yakoi","orcid":"https://orcid.org/0009-0006-9696-7276"},"institutions":[{"id":"https://openalex.org/I176808543","display_name":"Liaoning Technical University","ror":"https://ror.org/01n2bd587","country_code":"CN","type":"education","lineage":["https://openalex.org/I176808543"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Polycarp Shizawaliyi Yakoi","raw_affiliation_strings":["Liaoning Technical University, Huludao, Liaoning, China"],"raw_orcid":"https://orcid.org/0009-0006-9696-7276","affiliations":[{"raw_affiliation_string":"Liaoning Technical University, Huludao, Liaoning, China","institution_ids":["https://openalex.org/I176808543"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121569020","display_name":"Xiangfu Meng","orcid":null},"institutions":[{"id":"https://openalex.org/I176808543","display_name":"Liaoning Technical University","ror":"https://ror.org/01n2bd587","country_code":"CN","type":"education","lineage":["https://openalex.org/I176808543"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiangfu Meng","raw_affiliation_strings":["Liaoning Technical University, Huludao, Liaoning, China"],"raw_orcid":"https://orcid.org/0000-0001-7879-2368","affiliations":[{"raw_affiliation_string":"Liaoning Technical University, Huludao, Liaoning, China","institution_ids":["https://openalex.org/I176808543"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Chunlin Yu","orcid":"https://orcid.org/0009-0008-7364-046X"},"institutions":[{"id":"https://openalex.org/I176808543","display_name":"Liaoning Technical University","ror":"https://ror.org/01n2bd587","country_code":"CN","type":"education","lineage":["https://openalex.org/I176808543"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chunlin Yu","raw_affiliation_strings":["Liaoning Technical University, Huludao, Liaoning, China"],"raw_orcid":"https://orcid.org/0009-0008-7364-046X","affiliations":[{"raw_affiliation_string":"Liaoning Technical University, Huludao, Liaoning, China","institution_ids":["https://openalex.org/I176808543"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093419405","display_name":"Victor Adeyi Odeh","orcid":"https://orcid.org/0000-0001-6765-957X"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Victor Adeyi Odeh","raw_affiliation_strings":["School of Information and Communication Engineering, University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":"https://orcid.org/0000-0001-6765-957X","affiliations":[{"raw_affiliation_string":"School of Information and Communication Engineering, University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093027357","display_name":"Danladi Suleman","orcid":"https://orcid.org/0000-0002-9958-8320"},"institutions":[{"id":"https://openalex.org/I174025329","display_name":"University of the Sunshine Coast","ror":"https://ror.org/016gb9e15","country_code":"AU","type":"education","lineage":["https://openalex.org/I174025329"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Danladi Suleman","raw_affiliation_strings":["School of Science, Technology and Engineering, University of the Sunshine Coast, UniSC Moreton Bay, Sippy Downs1, QLD, Australia"],"raw_orcid":"https://orcid.org/0000-0002-9958-8320","affiliations":[{"raw_affiliation_string":"School of Science, Technology and Engineering, University of the Sunshine Coast, UniSC Moreton Bay, Sippy Downs1, QLD, Australia","institution_ids":["https://openalex.org/I174025329"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yongqin Zhang","orcid":"https://orcid.org/0009-0000-6687-6214"},"institutions":[{"id":"https://openalex.org/I176808543","display_name":"Liaoning Technical University","ror":"https://ror.org/01n2bd587","country_code":"CN","type":"education","lineage":["https://openalex.org/I176808543"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongqin Zhang","raw_affiliation_strings":["Liaoning Technical University, Huludao, Liaoning, China"],"raw_orcid":"https://orcid.org/0009-0000-6687-6214","affiliations":[{"raw_affiliation_string":"Liaoning Technical University, Huludao, Liaoning, China","institution_ids":["https://openalex.org/I176808543"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5121534714","display_name":"Xiaoyan Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I176808543","display_name":"Liaoning Technical University","ror":"https://ror.org/01n2bd587","country_code":"CN","type":"education","lineage":["https://openalex.org/I176808543"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoyan Zhang","raw_affiliation_strings":["Liaoning Technical University, Huludao, Liaoning, China"],"raw_orcid":"https://orcid.org/0000-0001-5260-1105","affiliations":[{"raw_affiliation_string":"Liaoning Technical University, Huludao, Liaoning, China","institution_ids":["https://openalex.org/I176808543"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5050895436"],"corresponding_institution_ids":["https://openalex.org/I176808543"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.60146375,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"14","issue":null,"first_page":"11505","last_page":"11533"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11344","display_name":"Traffic Prediction and Management Techniques","score":0.39419999718666077,"subfield":{"id":"https://openalex.org/subfields/2215","display_name":"Building and Construction"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11344","display_name":"Traffic Prediction and Management Techniques","score":0.39419999718666077,"subfield":{"id":"https://openalex.org/subfields/2215","display_name":"Building and Construction"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12205","display_name":"Time Series Analysis and Forecasting","score":0.20280000567436218,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.09939999878406525,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7211999893188477},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.5009999871253967},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.4413999915122986},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.4189999997615814},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.39719998836517334},{"id":"https://openalex.org/keywords/sparse-approximation","display_name":"Sparse approximation","score":0.3970000147819519},{"id":"https://openalex.org/keywords/time-series","display_name":"Time series","score":0.3939000070095062},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.3937999904155731},{"id":"https://openalex.org/keywords/sparse-matrix","display_name":"Sparse matrix","score":0.3917999863624573}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8486999869346619},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7211999893188477},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5378999710083008},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.5009999871253967},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.49129998683929443},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.4413999915122986},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.4189999997615814},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.39719998836517334},{"id":"https://openalex.org/C124066611","wikidata":"https://www.wikidata.org/wiki/Q28684319","display_name":"Sparse approximation","level":2,"score":0.3970000147819519},{"id":"https://openalex.org/C151406439","wikidata":"https://www.wikidata.org/wiki/Q186588","display_name":"Time series","level":2,"score":0.3939000070095062},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3937999904155731},{"id":"https://openalex.org/C56372850","wikidata":"https://www.wikidata.org/wiki/Q1050404","display_name":"Sparse matrix","level":3,"score":0.3917999863624573},{"id":"https://openalex.org/C12186640","wikidata":"https://www.wikidata.org/wiki/Q6815743","display_name":"Memory model","level":3,"score":0.3650999963283539},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.35740000009536743},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.3409000039100647},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32519999146461487},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3230000138282776},{"id":"https://openalex.org/C77637269","wikidata":"https://www.wikidata.org/wiki/Q7002051","display_name":"Neural coding","level":2,"score":0.31630000472068787},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.30889999866485596},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.29899999499320984},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.28940001130104065},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.2847999930381775},{"id":"https://openalex.org/C133488467","wikidata":"https://www.wikidata.org/wiki/Q6673524","display_name":"Long short term memory","level":4,"score":0.27619999647140503},{"id":"https://openalex.org/C143724316","wikidata":"https://www.wikidata.org/wiki/Q312468","display_name":"Series (stratigraphy)","level":2,"score":0.27149999141693115},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.27090001106262207},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.26750001311302185},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.2567000091075897}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/access.2025.3649490","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3649490","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1109/access.2025.3649490","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3649490","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W168746522","https://openalex.org/W2025768430","https://openalex.org/W2064675550","https://openalex.org/W2111406701","https://openalex.org/W2157331557","https://openalex.org/W2510642588","https://openalex.org/W2896457183","https://openalex.org/W2980025636","https://openalex.org/W3177318507","https://openalex.org/W4206420106","https://openalex.org/W4229604995","https://openalex.org/W4316467032","https://openalex.org/W4382203079","https://openalex.org/W4385245566","https://openalex.org/W4404783458","https://openalex.org/W4412405337","https://openalex.org/W4412405547"],"related_works":[],"abstract_inverted_index":{"Transformer-based":[0],"models":[1],"for":[2],"time":[3,67,79],"series":[4,68],"forecasting":[5],"have":[6],"advanced":[7],"considerably,":[8],"yet":[9],"many":[10],"approaches":[11],"treat":[12],"attention":[13,82],"sparsity":[14,133],"and":[15,29,46,62,134,152,166,180,186,193,208,231,238],"memory":[16,110,121,135],"mechanisms":[17],"as":[18],"separate":[19],"strategies.":[20],"In":[21],"this":[22],"extended":[23,142],"work,":[24],"we":[25],"revisit":[26],"these":[27,126],"paradigms":[28],"present":[30],"LTSMiTransformer,":[31],"a":[32,42,47,72,108,137,202],"unified":[33,48],"architecture":[34],"that":[35,51,198],"integrates":[36],"Learnable":[37],"Temporal":[38],"Sparse":[39],"Attention":[40],"(LTSA),":[41],"Memory-Augmented":[43],"Module":[44],"(MAM)":[45],"embedding":[49],"strategy":[50],"enhances":[52],"feature":[53],"representation":[54],"across":[55,210],"heterogeneous":[56],"datasets":[57,175,239],"to":[58,75,91,131,244],"improve":[59],"efficiency,":[60],"scalability,":[61,206],"generalization":[63,209],"in":[64,136],"long-horizon":[65],"multivariate":[66],"forecasting.":[69],"LTSA":[70],"employs":[71],"trainable":[73],"threshold":[74],"dynamically":[76],"filter":[77],"irrelevant":[78],"steps,":[80],"reducing":[81],"complexity":[83],"from":[84,176],"<italic":[85,92,99],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[86,89,93,96,100],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">O</i>":[87,94],"(<italic":[88,95],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">L</i>\u00b2)":[90],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">L</i>":[97],"log":[98],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">L</i>),":[101],"while":[102],"MAM":[103],"encodes":[104],"long-term":[105],"patterns":[106],"into":[107],"compact":[109],"bank":[111],"with":[112,170],"gated":[113],"updates,":[114],"enabling":[115],"persistent":[116],"context":[117],"retention":[118],"without":[119],"uncontrolled":[120],"growth.":[122],"Unlike":[123],"prior":[124],"models,":[125],"components":[127],"are":[128,240],"jointly":[129],"optimized":[130],"balance":[132,204],"coherent":[138],"training":[139],"framework.":[140],"This":[141,213],"work":[143],"provides":[144],"new":[145,225],"theoretical":[146,189,226],"analysis":[147],"of":[148],"LTSA\u2019s":[149],"sub-quadratic":[150],"efficiency":[151],"MAM\u2019s":[153],"convergence":[154],"behaviour":[155],"under":[156,162],"sparse":[157],"gating.":[158],"Comprehensive":[159],"robustness":[160,194],"tests":[161],"noise,":[163],"missing":[164],"values,":[165],"temporal":[167],"imbalance,":[168],"along":[169],"evaluations":[171,195],"on":[172,242],"eight":[173],"diverse":[174,211],"finance,":[177],"energy,":[178],"weather,":[179],"traffic,":[181],"demonstrate":[182],"LTSMiTransformer\u2019s":[183],"accuracy,":[184],"generalizability,":[185],"compactness.":[187],"The":[188],"proofs,":[190],"ablation":[191],"studies,":[192],"included":[196],"show":[197],"the":[199],"LTSMiTransformer":[200],"achieves":[201],"strong":[203],"between":[205],"interpretability,":[207],"datasets.":[212],"paper":[214],"significantly":[215],"expands":[216],"our":[217],"earlier":[218],"CNIOT":[219],"2025":[220],"conference":[221],"papers":[222],"by":[223],"adding":[224],"results,":[227],"deeper":[228],"joint-training":[229],"analysis,":[230],"wider":[232],"empirical":[233],"validation.":[234],"All":[235],"source":[236],"code":[237],"available":[241],"GitHub":[243],"support":[245],"reproducibility.":[246]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-12-30T00:00:00"}