{"id":"https://openalex.org/W7154384018","doi":"https://doi.org/10.1109/tpami.2026.3683792","title":"Neural Network Optimization Reimagined: Decoupled Techniques for Scratch and Fine-Tuning","display_name":"Neural Network Optimization Reimagined: Decoupled Techniques for Scratch and Fine-Tuning","publication_year":2026,"publication_date":"2026-04-14","ids":{"openalex":"https://openalex.org/W7154384018","doi":"https://doi.org/10.1109/tpami.2026.3683792","pmid":"https://pubmed.ncbi.nlm.nih.gov/41979964"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2026.3683792","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2026.3683792","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2604.22838","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133608457","display_name":"Xin Ning","orcid":"https://orcid.org/0000-0001-7897-1673"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Ning","raw_affiliation_strings":["Chinese Academy of Sciences"],"raw_orcid":"https://orcid.org/0000-0001-7897-1673","affiliations":[{"raw_affiliation_string":"Chinese Academy of Sciences","institution_ids":["https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101431986","display_name":"Qiankun Li","orcid":"https://orcid.org/0000-0001-5121-1682"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Qiankun Li","raw_affiliation_strings":["Nanyang Technological University"],"raw_orcid":"https://orcid.org/0000-0001-5121-1682","affiliations":[{"raw_affiliation_string":"Nanyang Technological University","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008207535","display_name":"Xiaolong Huang","orcid":"https://orcid.org/0009-0005-5502-2535"},"institutions":[{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Xiaolong Huang","raw_affiliation_strings":["Mila-Quebec AI Institute"],"raw_orcid":"https://orcid.org/0009-0005-5502-2535","affiliations":[{"raw_affiliation_string":"Mila-Quebec AI Institute","institution_ids":["https://openalex.org/I4210164802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021521914","display_name":"Qiupu Chen","orcid":"https://orcid.org/0000-0003-1036-6762"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qiupu Chen","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":"https://orcid.org/0000-0003-1036-6762","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090017573","display_name":"He Feng","orcid":"https://orcid.org/0000-0001-5059-4423"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Feng He","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":"https://orcid.org/0000-0001-5059-4423","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003551786","display_name":"Weijun Li","orcid":"https://orcid.org/0000-0001-9668-2883"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weijun Li","raw_affiliation_strings":["Chinese Academy of Sciences"],"raw_orcid":"https://orcid.org/0000-0001-9668-2883","affiliations":[{"raw_affiliation_string":"Chinese Academy of Sciences","institution_ids":["https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082046789","display_name":"Prayag Tiwari","orcid":"https://orcid.org/0000-0002-2851-4260"},"institutions":[{"id":"https://openalex.org/I746986","display_name":"Halmstad University","ror":"https://ror.org/03h0qfp10","country_code":"SE","type":"education","lineage":["https://openalex.org/I746986"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Prayag Tiwari","raw_affiliation_strings":["Halmstad University"],"raw_orcid":"https://orcid.org/0000-0002-2851-4260","affiliations":[{"raw_affiliation_string":"Halmstad University","institution_ids":["https://openalex.org/I746986"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5011205977","display_name":"X Y Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinwang Liu","raw_affiliation_strings":["National University of Defense Technology"],"raw_orcid":"https://orcid.org/0000-0001-9066-1475","affiliations":[{"raw_affiliation_string":"National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":65.6636,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.99881704,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"48","issue":"9","first_page":"10221","last_page":"10234"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.08460000157356262,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.08460000157356262,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.07859999686479568,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12611","display_name":"Neural Networks and Reservoir Computing","score":0.043699998408555984,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scratch","display_name":"Scratch","score":0.6784999966621399},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.6176000237464905},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.35010001063346863},{"id":"https://openalex.org/keywords/image-processing","display_name":"Image processing","score":0.29440000653266907},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.28459998965263367},{"id":"https://openalex.org/keywords/signal-processing","display_name":"Signal processing","score":0.28049999475479126}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.708899974822998},{"id":"https://openalex.org/C2781235140","wikidata":"https://www.wikidata.org/wiki/Q275131","display_name":"Scratch","level":2,"score":0.6784999966621399},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6176000237464905},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5378999710083008},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3517000079154968},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.35010001063346863},{"id":"https://openalex.org/C9417928","wikidata":"https://www.wikidata.org/wiki/Q1070689","display_name":"Image processing","level":3,"score":0.29440000653266907},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.28459998965263367},{"id":"https://openalex.org/C104267543","wikidata":"https://www.wikidata.org/wiki/Q208163","display_name":"Signal processing","level":3,"score":0.28049999475479126},{"id":"https://openalex.org/C175202392","wikidata":"https://www.wikidata.org/wiki/Q2434543","display_name":"Time delay neural network","level":3,"score":0.2786000072956085},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.2583000063896179},{"id":"https://openalex.org/C17500928","wikidata":"https://www.wikidata.org/wiki/Q959968","display_name":"Control system","level":2,"score":0.25769999623298645},{"id":"https://openalex.org/C155032097","wikidata":"https://www.wikidata.org/wiki/Q798503","display_name":"Backpropagation","level":3,"score":0.2531999945640564},{"id":"https://openalex.org/C124504099","wikidata":"https://www.wikidata.org/wiki/Q56933","display_name":"Image segmentation","level":3,"score":0.2517000138759613}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tpami.2026.3683792","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2026.3683792","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:41979964","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41979964","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":"Journal Article"},{"id":"pmh:oai:arXiv.org:2604.22838","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2604.22838","pdf_url":"https://arxiv.org/pdf/2604.22838","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:dr.ntu.edu.sg:10356/220989","is_oa":false,"landing_page_url":"https://hdl.handle.net/10356/220989","pdf_url":null,"source":{"id":"https://openalex.org/S4306402609","display_name":"DR-NTU (Nanyang Technological University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I172675005","host_organization_name":"Nanyang Technological University","host_organization_lineage":["https://openalex.org/I172675005"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Journal Article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2604.22838","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2604.22838","pdf_url":"https://arxiv.org/pdf/2604.22838","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G8748841592","display_name":null,"funder_award_id":"62373343","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7154384018.pdf","grobid_xml":"https://content.openalex.org/works/W7154384018.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"With":[0],"the":[1,6,12,45,55,103,120,135,154,161,168,191],"accumulation":[2],"of":[3,8,14,58,105,171,197],"resources":[4],"in":[5,17,134],"era":[7],"Big":[9],"Data":[10],"and":[11,98,108,140,147,187,194],"rise":[13],"pre-trained":[15,32],"models":[16,33],"deep":[18],"learning,":[19],"optimizing":[20],"neural":[21],"networks":[22],"for":[23,30,78],"various":[24],"tasks":[25],"often":[26],"involves":[27],"different":[28,172],"strategies":[29],"fine-tuning":[31,149],"versus":[34],"training":[35,81,84],"from":[36,85],"scratch.":[37],"However,":[38],"existing":[39],"optimizers":[40],"primarily":[41],"focus":[42],"on":[43],"reducing":[44],"loss":[46],"function":[47],"by":[48,100],"updating":[49],"model":[50],"parameters,":[51],"without":[52],"fully":[53],"addressing":[54],"unique":[56],"demands":[57,170],"these":[59,79],"two":[60],"major":[61],"paradigms.":[62],"In":[63],"this":[64],"paper,":[65],"we":[66,87,115,152],"propose":[67],"DualOpt,":[68],"a":[69,123],"novel":[70],"approach":[71],"that":[72],"decouples":[73],"optimization":[74],"techniques":[75],"specifically":[76],"tailored":[77],"distinct":[80],"scenarios.":[82],"For":[83,111],"scratch,":[86],"introduce":[88],"real-time":[89],"layer-wise":[90,155],"weight":[91,106,117,128,136,156],"decay,":[92],"designed":[93],"to":[94,158,167],"enhance":[95],"both":[96],"convergence":[97],"generalization":[99],"aligning":[101],"with":[102,119],"characteristics":[104],"updates":[107],"network":[109],"architecture.":[110],"more":[112],"importantly":[113],"fine-tuning,":[114],"integrate":[116],"rollback":[118,124,162],"optimizer,":[121],"incorporating":[122],"term":[125],"into":[126],"each":[127],"update":[129],"step.":[130],"This":[131],"ensures":[132],"consistency":[133],"distribution":[137],"between":[138],"upstream":[139],"downstream":[141,173],"models,":[142],"effectively":[143],"mitigating":[144],"knowledge":[145],"forgetting":[146],"improving":[148],"performance.":[150],"Additionally,":[151],"extend":[153],"decay":[157],"dynamically":[159],"adjust":[160],"levels":[163],"across":[164,177],"layers,":[165],"adapting":[166],"varying":[169],"tasks.":[174],"Extensive":[175],"experiments":[176],"diverse":[178],"tasks,":[179],"including":[180],"image":[181],"classification,":[182],"object":[183],"detection,":[184],"semantic":[185],"segmentation,":[186,189],"instance":[188],"demonstrate":[190],"broad":[192],"applicability":[193],"state-of-the-art":[195],"performance":[196],"DualOpt.":[198]},"counts_by_year":[{"year":2026,"cited_by_count":9}],"updated_date":"2026-08-18T07:49:30.821534","created_date":"2026-04-15T00:00:00"}