{"id":"https://openalex.org/W4384159580","doi":"https://doi.org/10.1109/access.2023.3295344","title":"Sparsity-Aware Orthogonal Initialization of Deep Neural Networks","display_name":"Sparsity-Aware Orthogonal Initialization of Deep Neural Networks","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4384159580","doi":"https://doi.org/10.1109/access.2023.3295344"},"language":"en","primary_location":{"id":"doi:10.1109/access.2023.3295344","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3295344","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10181312.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10181312.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001267478","display_name":"Kiara Esguerra","orcid":"https://orcid.org/0009-0006-5006-5672"},"institutions":[{"id":"https://openalex.org/I203899302","display_name":"Universiti Teknologi Petronas","ror":"https://ror.org/048g2sh07","country_code":"MY","type":"education","lineage":["https://openalex.org/I203899302"]}],"countries":["MY"],"is_corresponding":true,"raw_author_name":"Kiara Esguerra","raw_affiliation_strings":["Department of Electrical and Electronics Engineering, Universiti Teknologi PETRONAS, Seri Iskandar, Perak, Malaysia"],"affiliations":[{"raw_affiliation_string":"Department of Electrical and Electronics Engineering, Universiti Teknologi PETRONAS, Seri Iskandar, Perak, Malaysia","institution_ids":["https://openalex.org/I203899302"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051657405","display_name":"Muneeb Nasir","orcid":"https://orcid.org/0009-0008-7846-6480"},"institutions":[{"id":"https://openalex.org/I203899302","display_name":"Universiti Teknologi Petronas","ror":"https://ror.org/048g2sh07","country_code":"MY","type":"education","lineage":["https://openalex.org/I203899302"]}],"countries":["MY"],"is_corresponding":false,"raw_author_name":"Muneeb Nasir","raw_affiliation_strings":["Department of Electrical and Electronics Engineering, Universiti Teknologi PETRONAS, Seri Iskandar, Perak, Malaysia"],"affiliations":[{"raw_affiliation_string":"Department of Electrical and Electronics Engineering, Universiti Teknologi PETRONAS, Seri Iskandar, Perak, Malaysia","institution_ids":["https://openalex.org/I203899302"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034034713","display_name":"Tong Boon Tang","orcid":"https://orcid.org/0000-0002-5721-6828"},"institutions":[{"id":"https://openalex.org/I203899302","display_name":"Universiti Teknologi Petronas","ror":"https://ror.org/048g2sh07","country_code":"MY","type":"education","lineage":["https://openalex.org/I203899302"]}],"countries":["MY"],"is_corresponding":false,"raw_author_name":"Tong Boon Tang","raw_affiliation_strings":["Department of Electrical and Electronics Engineering, Universiti Teknologi PETRONAS, Seri Iskandar, Perak, Malaysia"],"affiliations":[{"raw_affiliation_string":"Department of Electrical and Electronics Engineering, Universiti Teknologi PETRONAS, Seri Iskandar, Perak, Malaysia","institution_ids":["https://openalex.org/I203899302"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048134093","display_name":"Afidalina Tumian","orcid":"https://orcid.org/0000-0003-2763-6677"},"institutions":[{"id":"https://openalex.org/I182023848","display_name":"Petronas (Malaysia)","ror":"https://ror.org/02tc7rm90","country_code":"MY","type":"company","lineage":["https://openalex.org/I182023848"]}],"countries":["MY"],"is_corresponding":false,"raw_author_name":"Afidalina Tumian","raw_affiliation_strings":["Petroliam Nasional Berhad (PETRONAS), Kuala Lumpur, Malaysia"],"affiliations":[{"raw_affiliation_string":"Petroliam Nasional Berhad (PETRONAS), Kuala Lumpur, Malaysia","institution_ids":["https://openalex.org/I182023848"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063934642","display_name":"Eric Tatt Wei Ho","orcid":"https://orcid.org/0000-0002-7590-1028"},"institutions":[{"id":"https://openalex.org/I203899302","display_name":"Universiti Teknologi Petronas","ror":"https://ror.org/048g2sh07","country_code":"MY","type":"education","lineage":["https://openalex.org/I203899302"]}],"countries":["MY"],"is_corresponding":false,"raw_author_name":"Eric Tatt Wei Ho","raw_affiliation_strings":["Department of Electrical and Electronics Engineering, Universiti Teknologi PETRONAS, Seri Iskandar, Perak, Malaysia"],"affiliations":[{"raw_affiliation_string":"Department of Electrical and Electronics Engineering, Universiti Teknologi PETRONAS, Seri Iskandar, Perak, Malaysia","institution_ids":["https://openalex.org/I203899302"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5001267478"],"corresponding_institution_ids":["https://openalex.org/I203899302"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.9644,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.69920123,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"11","issue":null,"first_page":"74165","last_page":"74181"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11809","display_name":"Advanced NMR Techniques and Applications","score":0.9768000245094299,"subfield":{"id":"https://openalex.org/subfields/1607","display_name":"Spectroscopy"},"field":{"id":"https://openalex.org/fields/16","display_name":"Chemistry"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11324","display_name":"Spectroscopy Techniques in Biomedical and Chemical Research","score":0.9607999920845032,"subfield":{"id":"https://openalex.org/subfields/1304","display_name":"Biophysics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.763235330581665},{"id":"https://openalex.org/keywords/initialization","display_name":"Initialization","score":0.6460326910018921},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5959392786026001},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49381908774375916},{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.4897775650024414},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4824591279029846},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.4477706253528595},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.35909610986709595}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.763235330581665},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.6460326910018921},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5959392786026001},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49381908774375916},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.4897775650024414},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4824591279029846},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.4477706253528595},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.35909610986709595},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C6557445","wikidata":"https://www.wikidata.org/wiki/Q173113","display_name":"Agronomy","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2023.3295344","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3295344","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10181312.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:2a2944aafcd943b5997428e3eabb8536","is_oa":true,"landing_page_url":"https://doaj.org/article/2a2944aafcd943b5997428e3eabb8536","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 11, Pp 74165-74181 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2023.3295344","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3295344","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10181312.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6059131399","display_name":null,"funder_award_id":"015LC0-403","funder_id":"https://openalex.org/F4320329394","funder_display_name":"Yayasan UTP"}],"funders":[{"id":"https://openalex.org/F4320323380","display_name":"Universiti Teknologi Petronas","ror":"https://ror.org/048g2sh07"},{"id":"https://openalex.org/F4320328058","display_name":"PETRONAS Research Sdn Bhd","ror":null},{"id":"https://openalex.org/F4320329394","display_name":"Yayasan UTP","ror":null},{"id":"https://openalex.org/F4320336394","display_name":"Science Education Institute, Department of Science and Technology, Republic of the Philippines","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4384159580.pdf","grobid_xml":"https://content.openalex.org/works/W4384159580.grobid-xml"},"referenced_works_count":92,"referenced_works":["https://openalex.org/W391578156","https://openalex.org/W1686810756","https://openalex.org/W1836465849","https://openalex.org/W2001339070","https://openalex.org/W2005644627","https://openalex.org/W2097117768","https://openalex.org/W2112796928","https://openalex.org/W2125930537","https://openalex.org/W2194775991","https://openalex.org/W2292929658","https://openalex.org/W2338343974","https://openalex.org/W2549139847","https://openalex.org/W2616463902","https://openalex.org/W2618530766","https://openalex.org/W2739309726","https://openalex.org/W2753358588","https://openalex.org/W2783538964","https://openalex.org/W2789210533","https://openalex.org/W2803968356","https://openalex.org/W2805003733","https://openalex.org/W2883059477","https://openalex.org/W2899962223","https://openalex.org/W2948161566","https://openalex.org/W2949736877","https://openalex.org/W2955425717","https://openalex.org/W2962990163","https://openalex.org/W2963163009","https://openalex.org/W2963477586","https://openalex.org/W2963664410","https://openalex.org/W2964065616","https://openalex.org/W2964116600","https://openalex.org/W2964537638","https://openalex.org/W3005578915","https://openalex.org/W3007700590","https://openalex.org/W3024929536","https://openalex.org/W3034222740","https://openalex.org/W3045233820","https://openalex.org/W3093317808","https://openalex.org/W3100809954","https://openalex.org/W3104688113","https://openalex.org/W3138851421","https://openalex.org/W3156066263","https://openalex.org/W3162340249","https://openalex.org/W3168746048","https://openalex.org/W3184606595","https://openalex.org/W3186482404","https://openalex.org/W4231543828","https://openalex.org/W4255256793","https://openalex.org/W4286898047","https://openalex.org/W4287204036","https://openalex.org/W4288096926","https://openalex.org/W4288796004","https://openalex.org/W4289293305","https://openalex.org/W4295151193","https://openalex.org/W4299997110","https://openalex.org/W4300167250","https://openalex.org/W4300971654","https://openalex.org/W4394639701","https://openalex.org/W6637373629","https://openalex.org/W6638667902","https://openalex.org/W6678818196","https://openalex.org/W6695533872","https://openalex.org/W6730172645","https://openalex.org/W6736987314","https://openalex.org/W6737988291","https://openalex.org/W6739659843","https://openalex.org/W6740275052","https://openalex.org/W6741844306","https://openalex.org/W6743969146","https://openalex.org/W6749180595","https://openalex.org/W6751707404","https://openalex.org/W6751979845","https://openalex.org/W6752356114","https://openalex.org/W6755174528","https://openalex.org/W6755310938","https://openalex.org/W6756052093","https://openalex.org/W6762718338","https://openalex.org/W6763914367","https://openalex.org/W6764345721","https://openalex.org/W6771579010","https://openalex.org/W6774676074","https://openalex.org/W6779460946","https://openalex.org/W6781016291","https://openalex.org/W6784331473","https://openalex.org/W6789913677","https://openalex.org/W6790050307","https://openalex.org/W6794426595","https://openalex.org/W6795113211","https://openalex.org/W6795537817","https://openalex.org/W6840012792","https://openalex.org/W6840989453","https://openalex.org/W6882249061"],"related_works":["https://openalex.org/W3204184292","https://openalex.org/W3176564347","https://openalex.org/W2355833770","https://openalex.org/W2951598131","https://openalex.org/W4221159459","https://openalex.org/W3128136716","https://openalex.org/W4226493464","https://openalex.org/W3133861977","https://openalex.org/W2951211570","https://openalex.org/W3103566983"],"abstract_inverted_index":{"Deep":[0],"neural":[1,80,123,151,216],"networks":[2,21,81,111,124,129,138,152,155,236],"have":[3],"achieved":[4],"impressive":[5],"pattern":[6],"recognition":[7],"and":[8,27,57,98,125,136,147,203,247],"generative":[9],"abilities":[10],"on":[11,183,192],"complex":[12],"tasks":[13],"by":[14,42],"developing":[15],"larger,":[16],"deeper":[17],"models,":[18],"but":[19,77,243],"these":[20,39],"are":[22,153,237],"increasingly":[23],"costly":[24],"to":[25,34,74,95,102,115,121,140,179,239],"train":[26],"implement.":[28],"There":[29],"is":[30,112,196],"in":[31,109,133],"tandem":[32],"interest":[33],"develop":[35],"sparse":[36,49,76,88,137,242],"versions":[37],"of":[38,61,234],"powerful":[40],"models":[41,218],"post-processing":[43],"with":[44,82,144,156,200,219],"weight":[45,249],"pruning":[46,132,161],"or":[47,230],"dynamic":[48],"training.":[50,117],"However,":[51],"it":[52,212],"requires":[53],"expensive":[54],"train-prune-finetune":[55],"cycles":[56],"compromises":[58],"the":[59,167],"trainability":[60],"very":[62,134],"deep":[63,135],"network":[64,89,217],"configurations.":[65],"We":[66,118],"introduce":[67],"sparsity-aware":[68],"orthogonal":[69,83,100,248],"initialization":[70],"(SAO),":[71],"a":[72,87,141,184],"method":[73],"initialize":[75],"maximally":[78,244],"connected":[79,245],"weights.":[84],"SAO":[85,110,120,128,154,168,172,195,235],"constructs":[86],"topology":[90,246],"leveraging":[91],"Ramanujan":[92],"expander":[93],"graphs":[94],"assure":[96],"connectivity":[97],"assigns":[99],"weights":[101],"attain":[103],"approximate":[104],"dynamical":[105],"isometry.":[106],"Network":[107],"sparsity":[108,169],"tunable":[113],"prior":[114,178],"model":[116,180],"compared":[119],"fully-connected":[122],"demonstrated":[126],"that":[127],"outperform":[130],"magnitude":[131],"up":[139],"thousand":[142],"layers":[143],"fewer":[145,220],"computations":[146],"training":[148,181],"iterations.":[149],"Convolutional":[150],"special":[157,225],"constraints,":[158],"while":[159],"kernel":[160],"may":[162,175],"be":[163,176],"interpreted":[164],"as":[165,207],"tuning":[166],"level.":[170],"Within":[171],"framework,":[173],"kernels":[174],"pruned":[177],"based":[182,191],"desired":[185],"compression":[186],"factor":[187],"rather":[188],"than":[189],"post-training":[190],"parameter-dependent":[193],"heuristics.":[194],"well-suited":[197],"for":[198],"applications":[199],"tight":[201],"energy":[202],"computation":[204],"budgets":[205],"such":[206],"edge":[208],"computing":[209],"tasks,":[210],"because":[211],"achieves":[213],"sparse,":[214],"trainable":[215],"learnable":[221],"parameters":[222],"without":[223],"requiring":[224],"layers,":[226],"additional":[227],"training,":[228],"scaling,":[229],"regularization.":[231],"The":[232],"advantages":[233],"attributed":[238],"both":[240],"its":[241],"initialization.":[250]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}