{"id":"https://openalex.org/W4399748876","doi":"https://doi.org/10.20944/preprints202406.1136.v1","title":"Brain-Inspired Sparse Training in MLP and Transformers with Network Science Modeling via Cannistraci-Hebb Soft Rule","display_name":"Brain-Inspired Sparse Training in MLP and Transformers with Network Science Modeling via Cannistraci-Hebb Soft Rule","publication_year":2024,"publication_date":"2024-06-17","ids":{"openalex":"https://openalex.org/W4399748876","doi":"https://doi.org/10.20944/preprints202406.1136.v1"},"language":"en","primary_location":{"id":"doi:10.20944/preprints202406.1136.v1","doi":"https://doi.org/10.20944/preprints202406.1136.v1","is_oa":true,"landing_page_url":"http://dx.doi.org/10.20944/preprints202406.1136.v1","pdf_url":"https://www.preprints.org/frontend/manuscript/1accd6cb60f953dabe5cc6e107c1b3f7/download_pub","source":{"id":"https://openalex.org/S6309402219","display_name":"Preprints.org","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.preprints.org/frontend/manuscript/1accd6cb60f953dabe5cc6e107c1b3f7/download_pub","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032429023","display_name":"Yingtao Zhang","orcid":"https://orcid.org/0000-0002-8587-9645","observed_orcids":["https://orcid.org/0000-0002-8587-9645"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yingtao Zhang","raw_affiliation_strings":["Center for Complex Network Intelligence (CCNI), Tsinghua Laboratory of Brain and Intelligence (THBI","Department of Computer Science","Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Complex Network Intelligence (CCNI), Tsinghua Laboratory of Brain and Intelligence (THBI","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Department of Computer Science","institution_ids":[]},{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009598811","display_name":"Jialin Zhao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jialin Zhao","raw_affiliation_strings":["Center for Complex Network Intelligence (CCNI), Tsinghua Laboratory of Brain and Intelligence (THBI","Department of Computer Science","Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Complex Network Intelligence (CCNI), Tsinghua Laboratory of Brain and Intelligence (THBI","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Department of Computer Science","institution_ids":[]},{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113106697","display_name":"Ziheng Liao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziheng Liao","raw_affiliation_strings":["Center for Complex Network Intelligence (CCNI), Tsinghua Laboratory of Brain and Intelligence (THBI","Department of Computer Science","Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Complex Network Intelligence (CCNI), Tsinghua Laboratory of Brain and Intelligence (THBI","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Department of Computer Science","institution_ids":[]},{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101083052","display_name":"Wenjing Wu","orcid":"https://orcid.org/0009-0008-4632-3506","observed_orcids":["https://orcid.org/0009-0008-4632-3506"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenjing Wu","raw_affiliation_strings":["Center for Complex Network Intelligence (CCNI), Tsinghua Laboratory of Brain and Intelligence (THBI","Department of Computer Science","Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Complex Network Intelligence (CCNI), Tsinghua Laboratory of Brain and Intelligence (THBI","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Department of Computer Science","institution_ids":[]},{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075734012","display_name":"Umberto Michieli","orcid":"https://orcid.org/0000-0003-2666-4342","observed_orcids":["https://orcid.org/0000-0003-2666-4342"]},"institutions":[{"id":"https://openalex.org/I138689650","display_name":"University of Padua","ror":"https://ror.org/00240q980","country_code":"IT","type":"education","lineage":["https://openalex.org/I138689650"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Umberto Michieli","raw_affiliation_strings":["University of Padova, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Padova, Italy","institution_ids":["https://openalex.org/I138689650"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007730744","display_name":"Carlo Vittorio Cannistraci","orcid":"https://orcid.org/0000-0003-0100-8410","observed_orcids":["https://orcid.org/0000-0003-0100-8410"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Carlo Vittorio Cannistraci","raw_affiliation_strings":["Center for Complex Network Intelligence (CCNI), Tsinghua Laboratory of Brain and Intelligence (THBI","Department of Biomedical Engineering","Department of Computer Science","Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0100-8410","affiliations":[{"raw_affiliation_string":"Center for Complex Network Intelligence (CCNI), Tsinghua Laboratory of Brain and Intelligence (THBI","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Department of Biomedical Engineering","institution_ids":[]},{"raw_affiliation_string":"Department of Computer Science","institution_ids":[]},{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5007730744"],"corresponding_institution_ids":["https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.8740000128746033,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.8740000128746033,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.0934000015258789,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.005100000184029341,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dynamic-sparse-training","display_name":"dynamic sparse training","score":0.968999981880188},{"id":"https://openalex.org/keywords/network-topology","display_name":"network topology","score":0.7289999723434448},{"id":"https://openalex.org/keywords/network-science","display_name":"network science","score":0.5540000200271606},{"id":"https://openalex.org/keywords/link-prediction","display_name":"link prediction","score":0.460999995470047},{"id":"https://openalex.org/keywords/small-world-networks","display_name":"small-world networks","score":0.6679999828338623},{"id":"https://openalex.org/keywords/scale-free-networks","display_name":"scale-free networks","score":0.984000027179718},{"id":"https://openalex.org/keywords/bipartite-graphs","display_name":"bipartite graphs","score":0.6690000295639038},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.5320000052452087},{"id":"https://openalex.org/keywords/image-classification","display_name":"image classification","score":0.6549999713897705},{"id":"https://openalex.org/keywords/machine-translation","display_name":"machine translation","score":0.9629999995231628},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.6129999756813049},{"id":"https://openalex.org/keywords/mlp","display_name":"MLP","score":0.335999995470047}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6293019652366638},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5824942588806152},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.562425971031189},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.511851966381073},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.45782196521759033},{"id":"https://openalex.org/C155032097","wikidata":"https://www.wikidata.org/wiki/Q798503","display_name":"Backpropagation","level":3,"score":0.44788169860839844},{"id":"https://openalex.org/C140073362","wikidata":"https://www.wikidata.org/wiki/Q738759","display_name":"Soft computing","level":3,"score":0.43746957182884216},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42713651061058044},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2121698260307312},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.08777019381523132},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0685034692287445}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.20944/preprints202406.1136.v1","doi":"https://doi.org/10.20944/preprints202406.1136.v1","is_oa":true,"landing_page_url":"http://dx.doi.org/10.20944/preprints202406.1136.v1","pdf_url":"https://www.preprints.org/frontend/manuscript/1accd6cb60f953dabe5cc6e107c1b3f7/download_pub","source":{"id":"https://openalex.org/S6309402219","display_name":"Preprints.org","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.20944/preprints202406.1136.v1","doi":"https://doi.org/10.20944/preprints202406.1136.v1","is_oa":true,"landing_page_url":"http://dx.doi.org/10.20944/preprints202406.1136.v1","pdf_url":"https://www.preprints.org/frontend/manuscript/1accd6cb60f953dabe5cc6e107c1b3f7/download_pub","source":{"id":"https://openalex.org/S6309402219","display_name":"Preprints.org","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4399748876.pdf","grobid_xml":"https://content.openalex.org/works/W4399748876.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W22297218","https://openalex.org/W2008620264","https://openalex.org/W2066692087","https://openalex.org/W2112090702","https://openalex.org/W2112796928","https://openalex.org/W2288135941","https://openalex.org/W2345720230","https://openalex.org/W2512924740","https://openalex.org/W2608761935","https://openalex.org/W2626778328","https://openalex.org/W2750384547","https://openalex.org/W2752820980","https://openalex.org/W2768501777","https://openalex.org/W2769826527","https://openalex.org/W2805003733","https://openalex.org/W2808133870","https://openalex.org/W2897624302","https://openalex.org/W2963909453","https://openalex.org/W3035180000","https://openalex.org/W3098372854","https://openalex.org/W3122998099","https://openalex.org/W3204130541","https://openalex.org/W3212680907","https://openalex.org/W4229005866","https://openalex.org/W4280556544","https://openalex.org/W4313484599","https://openalex.org/W4322718191","https://openalex.org/W4330337916","https://openalex.org/W4368755736","https://openalex.org/W4381586827","https://openalex.org/W4384918448","https://openalex.org/W4385800540","https://openalex.org/W4385805023","https://openalex.org/W4399051247","https://openalex.org/W4399053803"],"related_works":["https://openalex.org/W7196965313","https://openalex.org/W4414523259","https://openalex.org/W7105194766","https://openalex.org/W3208631787","https://openalex.org/W4388080212","https://openalex.org/W4399053803","https://openalex.org/W7202131505","https://openalex.org/W4284884593","https://openalex.org/W7221782583","https://openalex.org/W3035180000"],"abstract_inverted_index":{"Dynamic":[0],"sparse":[1,20,251,277],"training":[2,10,21,45,56,95,113,278],"is":[3],"an":[4,49],"effective":[5],"strategy":[6],"to":[7,36,53,85,122,221,232],"alleviate":[8],"the":[9,71,91,94,98,111,124,144,152,167,181,186,202,216,224,240,267],"and":[11,104,126,137,155,189,193],"inference":[12],"demands":[13],"of":[14,30,38,93,128,169,180,201,223,266],"artificial":[15],"neural":[16],"networks.":[17],"However,":[18,74],"current":[19],"methods":[22,279],"face":[23],"a":[24,62,119,133,229,246],"challenge":[25],"in":[26,57,280],"achieving":[27],"high":[28],"levels":[29],"sparsity":[31],"while":[32],"maintaining":[33],"performance":[34,168],"comparable":[35],"that":[37,163],"their":[39],"fully":[40,54,170,236],"connected":[41,55,171,237],"counterparts.":[42],"The":[43],"Cannistraci-Hebb":[44,112],"(CHT)":[46],"method":[47],"produces":[48],"ultra-sparse":[50],"advantage":[51],"compared":[52],"various":[58],"tasks":[59],"by":[60,176],"using":[61,148,177],"gradient-free":[63],"link":[64,80],"regrowth":[65,127],"method,":[66],"which":[67,117],"relies":[68],"solely":[69],"on":[70,79,185],"network":[72,99,139,145,158,219,259],"topology.":[73,140],"its":[75],"rigid":[76],"selection":[77],"based":[78],"prediction":[81],"scores":[82],"may":[83],"lead":[84],"epitopological":[86],"local":[87],"minima,":[88],"especially":[89],"at":[90],"beginning":[92],"process":[96],"when":[97],"topology":[100,146],"might":[101],"be":[102],"noisy":[103],"unreliable.":[105],"In":[106,206],"this":[107],"article,":[108],"we":[109,142,254],"introduce":[110],"soft":[114],"rule":[115],"(CHTs),":[116],"applies":[118],"flexible":[120],"approach":[121],"both":[123],"removal":[125],"links":[129,203],"during":[130],"training,":[131],"fostering":[132],"balance":[134],"between":[135],"exploring":[136],"exploiting":[138],"Additionally,":[141],"investigate":[143],"initialization":[147],"several":[149],"approaches,":[150],"including":[151],"bipartite":[153,156],"scale-free":[154],"small-world":[157],"models.":[159],"Empirical":[160],"results":[161,197],"show":[162],"CHTs":[164,213,272],"can":[165,194,214],"surpass":[166],"networks":[172],"with":[173,198,264],"MLP":[174],"architecture":[175],"only":[178,199],"1%":[179],"connections":[182,268],"(99%":[183],"sparsity)":[184],"MNIST,":[187],"EMNIST,":[188],"Fashion":[190],"MNIST":[191],"datasets":[192],"provide":[195],"remarkable":[196,230],"0.1%":[200],"(99.9%":[204],"sparsity).":[205],"some":[207],"MLPs":[208],"for":[209,249],"image":[210],"classification":[211],"tasks,":[212],"reduce":[215],"active":[217],"neuron":[218],"size":[220],"20%":[222],"original":[225],"nodes":[226],"(neurons),":[227],"demonstrating":[228],"ability":[231],"generalize":[233],"better":[234],"than":[235],"architectures,":[238],"reducing":[239],"entire":[241],"model":[242],"size.":[243],"This":[244],"represents":[245],"relevant":[247],"result":[248],"dynamic":[250,276],"training.":[252],"Finally,":[253],"present":[255],"evidence":[256],"from":[257],"larger":[258],"models":[260],"such":[261],"as":[262],"Transformers,":[263],"10%":[265],"(90%":[269],"sparsity),":[270],"where":[271],"outperform":[273],"other":[274],"prevalent":[275],"machine":[281],"translation":[282],"tasks.":[283]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2024-06-18T00:00:00"}