{"id":"https://openalex.org/W4200041270","doi":"https://doi.org/10.1109/tnnls.2021.3130756","title":"Parameter Efficient Neural Networks With Singular Value Decomposed Kernels","display_name":"Parameter Efficient Neural Networks With Singular Value Decomposed Kernels","publication_year":2021,"publication_date":"2021-12-23","ids":{"openalex":"https://openalex.org/W4200041270","doi":"https://doi.org/10.1109/tnnls.2021.3130756","pmid":"https://pubmed.ncbi.nlm.nih.gov/34941526"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2021.3130756","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tnnls.2021.3130756","pdf_url":"https://ieeexplore.ieee.org/ielx7/5962385/6104215/09662281.pdf","source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://ieeexplore.ieee.org/ielx7/5962385/6104215/09662281.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074882108","display_name":"David Vander Mijnsbrugge","orcid":"https://orcid.org/0000-0002-9918-3438"},"institutions":[{"id":"https://openalex.org/I32597200","display_name":"Ghent University","ror":"https://ror.org/00cv9y106","country_code":"BE","type":"education","lineage":["https://openalex.org/I32597200"]}],"countries":["BE"],"is_corresponding":true,"raw_author_name":"David Vander Mijnsbrugge","raw_affiliation_strings":["IDLab, Ghent University–imec, Gent, Belgium"],"affiliations":[{"raw_affiliation_string":"IDLab, Ghent University–imec, Gent, Belgium","institution_ids":["https://openalex.org/I32597200"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008111431","display_name":"Femke Ongenae","orcid":"https://orcid.org/0000-0003-2529-5477"},"institutions":[{"id":"https://openalex.org/I32597200","display_name":"Ghent University","ror":"https://ror.org/00cv9y106","country_code":"BE","type":"education","lineage":["https://openalex.org/I32597200"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Femke Ongenae","raw_affiliation_strings":["IDLab, Ghent University–imec, Gent, Belgium"],"affiliations":[{"raw_affiliation_string":"IDLab, Ghent University–imec, Gent, Belgium","institution_ids":["https://openalex.org/I32597200"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025315149","display_name":"Sofie Van Hoecke","orcid":"https://orcid.org/0000-0002-7865-6793"},"institutions":[{"id":"https://openalex.org/I32597200","display_name":"Ghent University","ror":"https://ror.org/00cv9y106","country_code":"BE","type":"education","lineage":["https://openalex.org/I32597200"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Sofie Van Hoecke","raw_affiliation_strings":["IDLab, Ghent University–imec, Gent, Belgium"],"affiliations":[{"raw_affiliation_string":"IDLab, Ghent University–imec, Gent, Belgium","institution_ids":["https://openalex.org/I32597200"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5074882108"],"corresponding_institution_ids":["https://openalex.org/I32597200"],"apc_list":null,"apc_paid":null,"fwci":0.699,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":{"value":0.7707865,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"34","issue":"9","first_page":"5682","last_page":"5692"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/stiefel-manifold","display_name":"Stiefel manifold","score":0.6821190118789673},{"id":"https://openalex.org/keywords/singular-value-decomposition","display_name":"Singular value decomposition","score":0.6785640716552734},{"id":"https://openalex.org/keywords/orthogonal-matrix","display_name":"Orthogonal matrix","score":0.6243594884872437},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.5845623016357422},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5521527528762817},{"id":"https://openalex.org/keywords/orthogonal-basis","display_name":"Orthogonal basis","score":0.5283496975898743},{"id":"https://openalex.org/keywords/matrix-multiplication","display_name":"Matrix multiplication","score":0.5259186029434204},{"id":"https://openalex.org/keywords/matrix","display_name":"Matrix (chemical analysis)","score":0.5004546642303467},{"id":"https://openalex.org/keywords/initialization","display_name":"Initialization","score":0.4828794300556183},{"id":"https://openalex.org/keywords/singular-value","display_name":"Singular value","score":0.47947439551353455},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.44741055369377136},{"id":"https://openalex.org/keywords/multiplication","display_name":"Multiplication (music)","score":0.4307646155357361},{"id":"https://openalex.org/keywords/manifold","display_name":"Manifold (fluid mechanics)","score":0.430084228515625},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.4141369163990021},{"id":"https://openalex.org/keywords/vector-space","display_name":"Vector space","score":0.4121996760368347},{"id":"https://openalex.org/keywords/applied-mathematics","display_name":"Applied mathematics","score":0.40917837619781494},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3043864965438843},{"id":"https://openalex.org/keywords/pure-mathematics","display_name":"Pure mathematics","score":0.20339879393577576},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.14124229550361633},{"id":"https://openalex.org/keywords/combinatorics","display_name":"Combinatorics","score":0.11304071545600891}],"concepts":[{"id":"https://openalex.org/C612670","wikidata":"https://www.wikidata.org/wiki/Q7616373","display_name":"Stiefel manifold","level":2,"score":0.6821190118789673},{"id":"https://openalex.org/C22789450","wikidata":"https://www.wikidata.org/wiki/Q420904","display_name":"Singular value decomposition","level":2,"score":0.6785640716552734},{"id":"https://openalex.org/C44292817","wikidata":"https://www.wikidata.org/wiki/Q333871","display_name":"Orthogonal matrix","level":3,"score":0.6243594884872437},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5845623016357422},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5521527528762817},{"id":"https://openalex.org/C187064257","wikidata":"https://www.wikidata.org/wiki/Q3306808","display_name":"Orthogonal basis","level":2,"score":0.5283496975898743},{"id":"https://openalex.org/C17349429","wikidata":"https://www.wikidata.org/wiki/Q1049914","display_name":"Matrix multiplication","level":3,"score":0.5259186029434204},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.5004546642303467},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.4828794300556183},{"id":"https://openalex.org/C109282560","wikidata":"https://www.wikidata.org/wiki/Q4166054","display_name":"Singular value","level":3,"score":0.47947439551353455},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.44741055369377136},{"id":"https://openalex.org/C2780595030","wikidata":"https://www.wikidata.org/wiki/Q3860309","display_name":"Multiplication (music)","level":2,"score":0.4307646155357361},{"id":"https://openalex.org/C529865628","wikidata":"https://www.wikidata.org/wiki/Q1790740","display_name":"Manifold (fluid mechanics)","level":2,"score":0.430084228515625},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4141369163990021},{"id":"https://openalex.org/C13336665","wikidata":"https://www.wikidata.org/wiki/Q125977","display_name":"Vector space","level":2,"score":0.4121996760368347},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.40917837619781494},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3043864965438843},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.20339879393577576},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.14124229550361633},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.11304071545600891},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C84114770","wikidata":"https://www.wikidata.org/wiki/Q46344","display_name":"Quantum","level":2,"score":0.0},{"id":"https://openalex.org/C158693339","wikidata":"https://www.wikidata.org/wiki/Q190524","display_name":"Eigenvalues and eigenvectors","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tnnls.2021.3130756","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tnnls.2021.3130756","pdf_url":"https://ieeexplore.ieee.org/ielx7/5962385/6104215/09662281.pdf","source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:34941526","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/34941526","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null},{"id":"pmh:oai:archive.ugent.be:01HCM6JC61QKEG5BXFE5EV55D9","is_oa":true,"landing_page_url":"http://hdl.handle.net/1854/LU-01HCM6JC61QKEG5BXFE5EV55D9","pdf_url":"https://biblio.ugent.be/publication/01HCM6JC61QKEG5BXFE5EV55D9/file/01HCM6MCDY8DT1J5ABQNP6C81B.pdf","source":{"id":"https://openalex.org/S4306400478","display_name":"Ghent University Academic Bibliography (Ghent University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I32597200","host_organization_name":"Ghent University","host_organization_lineage":["https://openalex.org/I32597200"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ISSN: 2162-2388","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"doi:10.1109/tnnls.2021.3130756","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tnnls.2021.3130756","pdf_url":"https://ieeexplore.ieee.org/ielx7/5962385/6104215/09662281.pdf","source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320313460","display_name":"Agentschap Innoveren en Ondernemen","ror":"https://ror.org/032xdry56"},{"id":"https://openalex.org/F4320327336","display_name":"Vlaamse regering","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4200041270.pdf","grobid_xml":"https://content.openalex.org/works/W4200041270.grobid-xml"},"referenced_works_count":48,"referenced_works":["https://openalex.org/W114517082","https://openalex.org/W613054787","https://openalex.org/W1522301498","https://openalex.org/W1529218607","https://openalex.org/W1539358435","https://openalex.org/W1600303801","https://openalex.org/W1994221311","https://openalex.org/W2003239136","https://openalex.org/W2018913979","https://openalex.org/W2039876715","https://openalex.org/W2126841255","https://openalex.org/W2194775991","https://openalex.org/W2414455940","https://openalex.org/W2552391848","https://openalex.org/W2555456875","https://openalex.org/W2556460356","https://openalex.org/W2557389081","https://openalex.org/W2611453176","https://openalex.org/W2800431972","https://openalex.org/W2898484532","https://openalex.org/W2942741533","https://openalex.org/W2951605425","https://openalex.org/W2952230511","https://openalex.org/W2963010571","https://openalex.org/W2963172539","https://openalex.org/W2963607709","https://openalex.org/W2970028551","https://openalex.org/W3004906533","https://openalex.org/W3088598409","https://openalex.org/W4229746864","https://openalex.org/W4294568934","https://openalex.org/W4301014524","https://openalex.org/W4385245566","https://openalex.org/W6631190155","https://openalex.org/W6632303579","https://openalex.org/W6649044840","https://openalex.org/W6683738474","https://openalex.org/W6729305679","https://openalex.org/W6730235532","https://openalex.org/W6734797129","https://openalex.org/W6737185050","https://openalex.org/W6738420126","https://openalex.org/W6739901393","https://openalex.org/W6741889403","https://openalex.org/W6748380982","https://openalex.org/W6755798685","https://openalex.org/W6762875675","https://openalex.org/W6771594235"],"related_works":["https://openalex.org/W1638580935","https://openalex.org/W2062388999","https://openalex.org/W3025932934","https://openalex.org/W2895619701","https://openalex.org/W2805026464","https://openalex.org/W2164903431","https://openalex.org/W4289422969","https://openalex.org/W3003561462","https://openalex.org/W1981489509","https://openalex.org/W4200041270"],"abstract_inverted_index":{"Traditionally,":[0],"neural":[1,122,137],"networks":[2],"are":[3,73,97,145,171],"viewed":[4],"from":[5,23],"the":[6,36,48,58,64,80,84,91,140,175,178,185],"perspective":[7],"of":[8,26,35,47,63,70,118,167,177,182,187],"connected":[9],"neuron":[10],"layers":[11,144],"represented":[12],"as":[13,33],"matrix":[14],"multiplications.":[15],"We":[16],"propose":[17],"to":[18,147,164,174],"compose":[19],"these":[20],"weight":[21,65,119,126],"matrices":[22,29,38,120,127],"a":[24,76,157,165],"set":[25],"orthogonal":[27,68,188],"basis":[28],"by":[30],"approaching":[31],"them":[32],"elements":[34],"real":[37],"vector":[39],"space":[40],"under":[41],"addition":[42],"and":[43,94,134,153,180],"multiplication.":[44],"Making":[45],"use":[46],"Kronecker":[49],"product":[50],"for":[51,90,101],"vectors,":[52],"this":[53,71,107],"composition":[54],"is":[55,109,161],"unified":[56],"with":[57,75],"singular":[59,92],"value":[60],"decomposition":[61],"(SVD)":[62],"matrix.":[66],"The":[67],"components":[69],"SVD":[72],"trained":[74],"descent":[77,104],"curve":[78],"on":[79,151],"Stiefel":[81],"manifold":[82,186],"using":[83,106],"Cayley":[85],"transform.":[86],"Next,":[87],"update":[88],"equations":[89],"values":[93],"initialization":[95],"routines":[96],"derived.":[98],"Finally,":[99],"acceleration":[100],"stochastic":[102],"gradient":[103],"optimization":[105,152,183],"formulation":[108],"discussed.":[110],"Our":[111],"proposed":[112],"method":[113,179],"allows":[114],"more":[115,135,141],"parameter-efficient":[116,142],"representations":[117],"in":[121,131],"networks.":[123],"These":[124,169],"decomposed":[125,143],"achieve":[128],"maximal":[129],"performance":[130],"both":[132],"standard":[133],"complicated":[136],"architectures.":[138],"Furthermore,":[139],"shown":[146],"be":[148],"less":[149],"dependent":[150],"better":[154],"conditioned.":[155],"As":[156],"tradeoff,":[158],"training":[159],"time":[160],"increased":[162],"up":[163],"factor":[166],"2.":[168],"observations":[170],"consequently":[172],"attributed":[173],"properties":[176],"choice":[181],"over":[184],"matrices.":[189]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2}],"updated_date":"2026-03-13T16:22:10.518609","created_date":"2025-10-10T00:00:00"}