{"id":"https://openalex.org/W2973164215","doi":"https://doi.org/10.1109/tnnls.2019.2935455","title":"Efficient Implementation of Second-Order Stochastic Approximation Algorithms in High-Dimensional Problems","display_name":"Efficient Implementation of Second-Order Stochastic Approximation Algorithms in High-Dimensional Problems","publication_year":2019,"publication_date":"2019-09-12","ids":{"openalex":"https://openalex.org/W2973164215","doi":"https://doi.org/10.1109/tnnls.2019.2935455","mag":"2973164215","pmid":"https://pubmed.ncbi.nlm.nih.gov/31536020"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2019.2935455","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2019.2935455","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045985355","display_name":"Jingyi Zhu","orcid":"https://orcid.org/0000-0001-7553-585X"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jingyi Zhu","raw_affiliation_strings":["Department of Applied Mathematics and Statistics, Johns Hopkins University, Baltimore, USA"],"raw_orcid":"https://orcid.org/0000-0001-7553-585X","affiliations":[{"raw_affiliation_string":"Department of Applied Mathematics and Statistics, Johns Hopkins University, Baltimore, USA","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100447539","display_name":"Long Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Long Wang","raw_affiliation_strings":["Department of Applied Mathematics and Statistics, Johns Hopkins University, Baltimore, USA"],"raw_orcid":"https://orcid.org/0000-0001-6328-8222","affiliations":[{"raw_affiliation_string":"Department of Applied Mathematics and Statistics, Johns Hopkins University, Baltimore, USA","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066890566","display_name":"James C. Spall","orcid":"https://orcid.org/0000-0001-8034-8864"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I2802946424","display_name":"Johns Hopkins University Applied Physics Laboratory","ror":"https://ror.org/029pp9z10","country_code":"US","type":"facility","lineage":["https://openalex.org/I145311948","https://openalex.org/I2802946424"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James C. Spall","raw_affiliation_strings":["Applied Physics Laboratory, Johns Hopkins University, Laurel, USA","Department of Applied Mathematics and Statistics, Johns Hopkins University, Baltimore, USA"],"raw_orcid":"https://orcid.org/0000-0001-8034-8864","affiliations":[{"raw_affiliation_string":"Applied Physics Laboratory, Johns Hopkins University, Laurel, USA","institution_ids":["https://openalex.org/I2802946424"]},{"raw_affiliation_string":"Department of Applied Mathematics and Statistics, Johns Hopkins University, Baltimore, USA","institution_ids":["https://openalex.org/I145311948"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.2707,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.85303973,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"31","issue":"8","first_page":"3087","last_page":"3099"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10500","display_name":"Sparse and Compressive Sensing Techniques","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.6625530123710632},{"id":"https://openalex.org/keywords/simultaneous-perturbation-stochastic-approximation","display_name":"Simultaneous perturbation stochastic approximation","score":0.646312415599823},{"id":"https://openalex.org/keywords/flops","display_name":"FLOPS","score":0.5939653515815735},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.5665120482444763},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.5303524136543274},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.49595722556114197},{"id":"https://openalex.org/keywords/factorization","display_name":"Factorization","score":0.49171334505081177},{"id":"https://openalex.org/keywords/order","display_name":"Order (exchange)","score":0.46950507164001465},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.40870168805122375},{"id":"https://openalex.org/keywords/discrete-mathematics","display_name":"Discrete mathematics","score":0.3672977685928345},{"id":"https://openalex.org/keywords/applied-mathematics","display_name":"Applied mathematics","score":0.36028262972831726},{"id":"https://openalex.org/keywords/combinatorics","display_name":"Combinatorics","score":0.3542443513870239},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.27549469470977783},{"id":"https://openalex.org/keywords/stochastic-process","display_name":"Stochastic process","score":0.22960978746414185},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.10422363877296448},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.10080122947692871},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.09441244602203369}],"concepts":[{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.6625530123710632},{"id":"https://openalex.org/C2779880469","wikidata":"https://www.wikidata.org/wiki/Q17084424","display_name":"Simultaneous perturbation stochastic approximation","level":3,"score":0.646312415599823},{"id":"https://openalex.org/C3826847","wikidata":"https://www.wikidata.org/wiki/Q188768","display_name":"FLOPS","level":2,"score":0.5939653515815735},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5665120482444763},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.5303524136543274},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49595722556114197},{"id":"https://openalex.org/C187834632","wikidata":"https://www.wikidata.org/wiki/Q188804","display_name":"Factorization","level":2,"score":0.49171334505081177},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.46950507164001465},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.40870168805122375},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.3672977685928345},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.36028262972831726},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.3542443513870239},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27549469470977783},{"id":"https://openalex.org/C8272713","wikidata":"https://www.wikidata.org/wiki/Q176737","display_name":"Stochastic process","level":2,"score":0.22960978746414185},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.10422363877296448},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.10080122947692871},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.09441244602203369},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2019.2935455","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2019.2935455","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:31536020","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/31536020","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320310145","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":49,"referenced_works":["https://openalex.org/W359561291","https://openalex.org/W1491468723","https://openalex.org/W1491622225","https://openalex.org/W1521291534","https://openalex.org/W1521333973","https://openalex.org/W1522301498","https://openalex.org/W1625941627","https://openalex.org/W1982672762","https://openalex.org/W1995393679","https://openalex.org/W2016814974","https://openalex.org/W2031392034","https://openalex.org/W2036509408","https://openalex.org/W2058221770","https://openalex.org/W2067186191","https://openalex.org/W2070709745","https://openalex.org/W2091638851","https://openalex.org/W2097115711","https://openalex.org/W2107438106","https://openalex.org/W2109722160","https://openalex.org/W2114123905","https://openalex.org/W2119513700","https://openalex.org/W2124289529","https://openalex.org/W2128452997","https://openalex.org/W2154987621","https://openalex.org/W2155894447","https://openalex.org/W2166347285","https://openalex.org/W2168658406","https://openalex.org/W2171562370","https://openalex.org/W2189431025","https://openalex.org/W2217490359","https://openalex.org/W2344249156","https://openalex.org/W2493209382","https://openalex.org/W2565654137","https://openalex.org/W2610857016","https://openalex.org/W2885971921","https://openalex.org/W2956209607","https://openalex.org/W2963644203","https://openalex.org/W2963941964","https://openalex.org/W2964121744","https://openalex.org/W3029645440","https://openalex.org/W4234369034","https://openalex.org/W4301014524","https://openalex.org/W6629379589","https://openalex.org/W6631190155","https://openalex.org/W6636588491","https://openalex.org/W6676105031","https://openalex.org/W6677572203","https://openalex.org/W6683107984","https://openalex.org/W6687082314"],"related_works":["https://openalex.org/W2895097035","https://openalex.org/W4206903459","https://openalex.org/W2754816816","https://openalex.org/W4366280654","https://openalex.org/W3160167280","https://openalex.org/W3020853991","https://openalex.org/W4362706668","https://openalex.org/W4231621013","https://openalex.org/W3171021120","https://openalex.org/W3008318776"],"abstract_inverted_index":{"Stochastic":[0,25],"approximation":[1,54],"(SA)":[2],"algorithms":[3,146],"have":[4],"been":[5],"widely":[6],"applied":[7],"in":[8,65,125,196,203],"minimization":[9],"problems":[10],"when":[11],"the":[12,16,39,49,57,78,82,88,100,103,108,119,128,144,148,156,183,191],"loss":[13],"functions":[14],"and/or":[15],"gradient":[17,26,60],"information":[18],"are":[19,62,187],"only":[20],"accessible":[21],"through":[22],"noisy":[23,131],"evaluations.":[24,133],"(SG)":[27],"descent-a":[28],"first-order":[29],"algorithm":[30],"and":[31,56,72,153,178,198],"a":[32,139],"workhorse":[33],"of":[34,43,87,102,127,130,180],"much":[35],"machine":[36],"learning-is":[37],"perhaps":[38],"most":[40],"famous":[41],"form":[42],"SA.":[44],"Among":[45],"all":[46],"SA":[47],"algorithms,":[48],"second-order":[50,58],"simultaneous":[51],"perturbation":[52],"stochastic":[53,59],"(2SPSA)":[55],"(2SG)":[61],"particularly":[63],"efficient":[64],"handling":[66],"highdimensional":[67],"problems,":[68],"covering":[69],"both":[70],"gradient-free":[71],"gradientbased":[73],"scenarios.":[74],"However,":[75],"due":[76],"to":[77,141,167],"necessary":[79],"matrix":[80,151],"operations,":[81],"per-iteration":[83,122],"floating-point-operations":[84],"(FLOPs)":[85],"cost":[86,115,124,158],"standard":[89,192],"2SPSA/2SG":[90,145],"is":[91,99,116,159,201],"O(p":[92,109,162,168],"<sup":[93,110,163,169],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[94,111,164,170],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">3</sup>":[95,112,165],"),":[96],"where":[97],"p":[98],"dimension":[101],"underlying":[104],"parameter.":[105],"Note":[106],"that":[107,155],")":[113,166],"FLOPs":[114,157],"distinct":[117],"from":[118,161,190],"classical":[120],"SPSA-based":[121],"O(1)":[123],"terms":[126],"number":[129],"function":[132],"In":[134],"this":[135],"work,":[136],"we":[137],"propose":[138],"technique":[140],"efficiently":[142],"implement":[143],"via":[147],"symmetric":[149],"indefinite":[150],"factorization":[152],"show":[154],"reduced":[160],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">2</sup>":[171],").":[172],"The":[173,194],"formal":[174],"almost":[175],"sure":[176],"convergence":[177,181],"rate":[179],"for":[182],"newly":[184],"proposed":[185],"approach":[186],"directly":[188],"inherited":[189],"2SPSA/2SG.":[193],"improvement":[195],"efficiency":[197],"numerical":[199,205],"stability":[200],"demonstrated":[202],"two":[204],"studies.":[206]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}