{"id":"https://openalex.org/W4385768058","doi":"https://doi.org/10.24963/ijcai.2023/493","title":"Singularformer: Learning to Decompose Self-Attention to Linearize the Complexity of Transformer","display_name":"Singularformer: Learning to Decompose Self-Attention to Linearize the Complexity of Transformer","publication_year":2023,"publication_date":"2023-08-01","ids":{"openalex":"https://openalex.org/W4385768058","doi":"https://doi.org/10.24963/ijcai.2023/493"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2023/493","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2023/493","pdf_url":"https://www.ijcai.org/proceedings/2023/0493.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.ijcai.org/proceedings/2023/0493.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020044791","display_name":"Yifan Wu","orcid":"https://orcid.org/0000-0002-2856-9835"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yifan Wu","raw_affiliation_strings":["Central South University","The School of Computer Science and Engineering, Central South University, Changsha, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University","institution_ids":["https://openalex.org/I139660479"]},{"raw_affiliation_string":"The School of Computer Science and Engineering, Central South University, Changsha, China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025341442","display_name":"Shichao Kan","orcid":"https://orcid.org/0000-0003-0097-6196"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shichao Kan","raw_affiliation_strings":["Central South University","The School of Computer Science and Engineering, Central South University, Changsha, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University","institution_ids":["https://openalex.org/I139660479"]},{"raw_affiliation_string":"The School of Computer Science and Engineering, Central South University, Changsha, China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033885404","display_name":"Min Zeng","orcid":"https://orcid.org/0000-0002-1726-0955"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Zeng","raw_affiliation_strings":["Central South University","The School of Computer Science and Engineering, Central South University, Changsha, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University","institution_ids":["https://openalex.org/I139660479"]},{"raw_affiliation_string":"The School of Computer Science and Engineering, Central South University, Changsha, China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100719013","display_name":"Min Li","orcid":"https://orcid.org/0000-0002-0188-1394"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Li","raw_affiliation_strings":["Central South University","The School of Computer Science and Engineering, Central South University, Changsha, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University","institution_ids":["https://openalex.org/I139660479"]},{"raw_affiliation_string":"The School of Computer Science and Engineering, Central South University, Changsha, China","institution_ids":["https://openalex.org/I139660479"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I139660479"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4433","last_page":"4441"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/singular-value-decomposition","display_name":"Singular value decomposition","score":0.7189614176750183},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7081469297409058},{"id":"https://openalex.org/keywords/matrix-multiplication","display_name":"Matrix multiplication","score":0.5730606317520142},{"id":"https://openalex.org/keywords/associative-property","display_name":"Associative property","score":0.5675258636474609},{"id":"https://openalex.org/keywords/computational-complexity-theory","display_name":"Computational complexity theory","score":0.508415162563324},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.5070035457611084},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.49140751361846924},{"id":"https://openalex.org/keywords/quadratic-equation","display_name":"Quadratic equation","score":0.4830373227596283},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.44908612966537476},{"id":"https://openalex.org/keywords/matrix-decomposition","display_name":"Matrix decomposition","score":0.4412175416946411},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.43414193391799927},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.41475972533226013},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37645164132118225},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.3564434051513672},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.17456313967704773},{"id":"https://openalex.org/keywords/quantum","display_name":"Quantum","score":0.1075391173362732}],"concepts":[{"id":"https://openalex.org/C22789450","wikidata":"https://www.wikidata.org/wiki/Q420904","display_name":"Singular value decomposition","level":2,"score":0.7189614176750183},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7081469297409058},{"id":"https://openalex.org/C17349429","wikidata":"https://www.wikidata.org/wiki/Q1049914","display_name":"Matrix multiplication","level":3,"score":0.5730606317520142},{"id":"https://openalex.org/C159423971","wikidata":"https://www.wikidata.org/wiki/Q177251","display_name":"Associative property","level":2,"score":0.5675258636474609},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.508415162563324},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5070035457611084},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.49140751361846924},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.4830373227596283},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.44908612966537476},{"id":"https://openalex.org/C42355184","wikidata":"https://www.wikidata.org/wiki/Q1361088","display_name":"Matrix decomposition","level":3,"score":0.4412175416946411},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.43414193391799927},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.41475972533226013},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37645164132118225},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3564434051513672},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.17456313967704773},{"id":"https://openalex.org/C84114770","wikidata":"https://www.wikidata.org/wiki/Q46344","display_name":"Quantum","level":2,"score":0.1075391173362732},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C158693339","wikidata":"https://www.wikidata.org/wiki/Q190524","display_name":"Eigenvalues and eigenvectors","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.24963/ijcai.2023/493","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2023/493","pdf_url":"https://www.ijcai.org/proceedings/2023/0493.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2023/493","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2023/493","pdf_url":"https://www.ijcai.org/proceedings/2023/0493.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.7300000190734863}],"awards":[{"id":"https://openalex.org/G3161298457","display_name":null,"funder_award_id":"No. 61832019","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4043378463","display_name":null,"funder_award_id":"61832019","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321514","display_name":"Central South University","ror":"https://ror.org/00f1zfq44"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385768058.pdf"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W1493526108","https://openalex.org/W1502916507","https://openalex.org/W2100074460","https://openalex.org/W2113459411","https://openalex.org/W2129250947","https://openalex.org/W2161072217","https://openalex.org/W2396881363","https://openalex.org/W2763421725","https://openalex.org/W2896457183","https://openalex.org/W2908510526","https://openalex.org/W2965373594","https://openalex.org/W3015468748","https://openalex.org/W3033529678","https://openalex.org/W3034772996","https://openalex.org/W3037798801","https://openalex.org/W3094502228","https://openalex.org/W3118608800","https://openalex.org/W3120633509","https://openalex.org/W3138516171","https://openalex.org/W3176948526","https://openalex.org/W3177828909","https://openalex.org/W3181262653","https://openalex.org/W3213305054","https://openalex.org/W4221157572","https://openalex.org/W4226261765","https://openalex.org/W4287704453","https://openalex.org/W4295838474","https://openalex.org/W4296932804","https://openalex.org/W4309793872","https://openalex.org/W4323654151","https://openalex.org/W4385245566"],"related_works":["https://openalex.org/W2782904003","https://openalex.org/W4226434912","https://openalex.org/W2118633810","https://openalex.org/W2150953077","https://openalex.org/W2002598339","https://openalex.org/W2920931670","https://openalex.org/W1995410415","https://openalex.org/W2133814741","https://openalex.org/W3108343813","https://openalex.org/W4312481107"],"abstract_inverted_index":{"Transformers":[0],"achieve":[1],"excellent":[2],"performance":[3],"in":[4,121],"a":[5,38,60,131],"variety":[6],"of":[7,54,76,98,107],"domains":[8],"since":[9],"they":[10],"can":[11],"capture":[12],"long-distance":[13],"dependencies":[14],"through":[15],"the":[16,49,55,70,74,95,105,122,137,155],"self-attention":[17,20,65,116,133],"mechanism.":[18,66],"However,":[19],"is":[21,101,168],"computationally":[22],"costly":[23],"due":[24],"to":[25,47,58,89,103,114],"its":[26],"quadratic":[27],"complexity":[28],"and":[29,62,85,124,162],"high":[30],"memory":[31],"consumption.":[32],"In":[33],"this":[34],"paper,":[35],"we":[36,68],"propose":[37],"novel":[39],"Transformer":[40,157],"variant":[41],"(Singularformer)":[42],"that":[43,150],"uses":[44],"neural":[45,87],"networks":[46,88],"learn":[48,90],"singular":[50,82],"value":[51,83],"decomposition":[52,84],"process":[53,135],"attention":[56,71],"matrix":[57,72,78,92,99],"design":[59,86],"linear-complexity":[61],"memory-efficient":[63],"global":[64],"Specifically,":[67],"decompose":[69],"into":[73],"product":[75],"three":[77],"factors":[79],"based":[80],"on":[81,136,145],"these":[91],"factors,":[93],"then":[94],"associative":[96],"law":[97],"multiplication":[100],"used":[102],"linearize":[104],"calculation":[106],"self-attention.":[108],"The":[109],"above":[110],"procedure":[111],"allows":[112],"us":[113],"compute":[115],"as":[117],"two-dimensional":[118],"reduction":[119],"processes":[120],"first":[123,138],"second":[125],"token":[126,141],"dimensional":[127,139],"spaces,":[128],"followed":[129],"by":[130],"multi-head":[132],"computational":[134],"reduced":[140],"features.":[142],"Experimental":[143],"results":[144],"8":[146],"real-world":[147],"datasets":[148],"demonstrate":[149],"Singularformer":[151],"performs":[152],"favorably":[153],"against":[154],"other":[156],"variants":[158],"with":[159],"lower":[160],"time":[161],"space":[163],"complexity.":[164],"Our":[165],"source":[166],"code":[167],"publicly":[169],"available":[170],"at":[171],"https://github.com/CSUBioGroup/Singularformer.":[172]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":4}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}