{"id":"https://openalex.org/W7204211272","doi":"https://doi.org/10.25781/kaust-zxu1e","title":"Understanding Practical Optimization in Machine Learning","display_name":"Understanding Practical Optimization in Machine Learning","publication_year":2026,"publication_date":"2026-08-25","ids":{"openalex":"https://openalex.org/W7204211272","doi":"https://doi.org/10.25781/kaust-zxu1e"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.35209","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.35209","pdf_url":"https://arxiv.org/pdf/2610.35209","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.35209","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5025477585","display_name":"Egor Shulgin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I71920554","display_name":"King Abdullah University of Science and Technology","ror":"https://ror.org/01q3tbs38","country_code":"SA","type":"education","lineage":["https://openalex.org/I71920554"]}],"countries":["SA"],"is_corresponding":true,"raw_author_name":"Shulgin, Egor","raw_affiliation_strings":["King Abdullah University of Science and Technology (KAUST), Thuwal, Saudi Arabia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"King Abdullah University of Science and Technology (KAUST), Thuwal, Saudi Arabia","institution_ids":["https://openalex.org/I71920554"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5025477585"],"corresponding_institution_ids":["https://openalex.org/I71920554"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dropout","display_name":"dropout","score":0.5640000104904175},{"id":"https://openalex.org/keywords/sparse-training","display_name":"sparse training","score":0.6520000100135803},{"id":"https://openalex.org/keywords/adaptive-clipping","display_name":"adaptive clipping","score":0.5360000133514404},{"id":"https://openalex.org/keywords/convergence-guarantees","display_name":"convergence guarantees","score":0.7099999785423279},{"id":"https://openalex.org/keywords/generalized-smoothness","display_name":"generalized smoothness","score":0.6899999976158142},{"id":"https://openalex.org/keywords/muon-optimizer","display_name":"Muon optimizer","score":0.7829999923706055},{"id":"https://openalex.org/keywords/momentum-method","display_name":"momentum methods","score":0.38600000739097595},{"id":"https://openalex.org/keywords/hyperparameter-sensitivity","display_name":"hyperparameter sensitivity","score":0.7680000066757202}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6801999807357788},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5310999751091003},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5248000025749207},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4787999987602234},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.43149998784065247},{"id":"https://openalex.org/C2780695315","wikidata":"https://www.wikidata.org/wiki/Q3799040","display_name":"Unobservable","level":2,"score":0.42969998717308044},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.4043000042438507},{"id":"https://openalex.org/C47559304","wikidata":"https://www.wikidata.org/wiki/Q1702189","display_name":"Orthogonalization","level":2,"score":0.3977000117301941},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.3871000111103058},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.382999986410141},{"id":"https://openalex.org/C102634674","wikidata":"https://www.wikidata.org/wiki/Q868473","display_name":"Smoothness","level":2,"score":0.37540000677108765},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.3725000023841858},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.36570000648498535},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.3481000065803528},{"id":"https://openalex.org/C2776848632","wikidata":"https://www.wikidata.org/wiki/Q853463","display_name":"Clipping (morphology)","level":2,"score":0.34610000252723694},{"id":"https://openalex.org/C43711488","wikidata":"https://www.wikidata.org/wiki/Q7534783","display_name":"Skew","level":2,"score":0.3111000061035156},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3086000084877014},{"id":"https://openalex.org/C118671147","wikidata":"https://www.wikidata.org/wiki/Q578714","display_name":"Quantile","level":2,"score":0.30709999799728394},{"id":"https://openalex.org/C3770464","wikidata":"https://www.wikidata.org/wiki/Q775963","display_name":"Smoothing","level":2,"score":0.3000999987125397},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.29899999499320984},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.2976999878883362},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.2703999876976013},{"id":"https://openalex.org/C116149140","wikidata":"https://www.wikidata.org/wiki/Q2070951","display_name":"Descent direction","level":4,"score":0.26820001006126404},{"id":"https://openalex.org/C2778584072","wikidata":"https://www.wikidata.org/wiki/Q7353545","display_name":"Robustification","level":3,"score":0.2669999897480011},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2540000081062317},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.2540000081062317},{"id":"https://openalex.org/C73000952","wikidata":"https://www.wikidata.org/wiki/Q17007827","display_name":"Discretization","level":2,"score":0.25209999084472656},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.25189998745918274}],"mesh":[],"locations_count":4,"locations":[{"id":"pmh:oai:arXiv.org:2610.35209","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.35209","pdf_url":"https://arxiv.org/pdf/2610.35209","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:doi:10.25781/kaust-zxu1e","doi":null,"is_oa":false,"landing_page_url":"https://repository.kaust.edu.sa/bitstreams/8d3c7cad-3bc0-43e9-bba7-01056e3aae5f/download","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dissertation"},{"id":"doi:10.48550/arxiv.2610.35209","doi":"https://doi.org/10.48550/arxiv.2610.35209","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.35209","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.25781/kaust-zxu1e","doi":"https://doi.org/10.25781/kaust-zxu1e","is_oa":true,"landing_page_url":"https://doi.org/10.25781/kaust-zxu1e","pdf_url":null,"source":{"id":"https://openalex.org/S4306401596","display_name":"King Abdullah University of Science and Technology Repository (King Abdullah University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I71920554","host_organization_name":"King Abdullah University of Science and Technology","host_organization_lineage":["https://openalex.org/I71920554"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dissertation"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.35209","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.35209","pdf_url":"https://arxiv.org/pdf/2610.35209","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204211272.pdf","grobid_xml":"https://content.openalex.org/works/W7204211272.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4417298428","https://openalex.org/W7015074232","https://openalex.org/W4389116634","https://openalex.org/W7004807045","https://openalex.org/W7116917668","https://openalex.org/W6884720798","https://openalex.org/W7114831471","https://openalex.org/W2922582589","https://openalex.org/W835081201","https://openalex.org/W7167227056"],"abstract_inverted_index":{"Modern":[0],"machine":[1],"learning":[2,201],"relies":[3],"on":[4,150,165],"practical":[5,204],"optimization":[6,217],"methods":[7,205],"that":[8,58,105,178,214],"often":[9],"differ":[10],"from":[11],"the":[12,37,74,77,85,167,179,200,212],"algorithms":[13],"studied":[14],"in":[15,182],"classical":[16],"analyses:":[17],"they":[18],"train":[19],"submodels,":[20],"clip":[21],"or":[22,29],"normalize":[23],"gradients,":[24],"use":[25],"layer-wise":[26,159],"non-Euclidean":[27],"directions,":[28],"approximate":[30,180],"costly":[31],"inner":[32],"computations.":[33],"This":[34],"thesis":[35],"analyzes":[36],"objective,":[38,90],"assumptions,":[39,142],"update,":[40],"and":[41,96,134,156,163,172],"computation":[42],"defined":[43],"by":[44],"these":[45],"procedures,":[46],"rather":[47],"than":[48],"those":[49],"of":[50],"simplified":[51],"proxies.":[52],"For":[53,100,147],"submodel":[54],"training,":[55],"we":[56,103,154],"show":[57,104,177],"Independent":[59],"Subnetwork":[60],"Training":[61,82],"can":[62],"converge":[63],"to":[64],"a":[65,158],"shifted":[66],"fixed":[67,113],"point":[68],"because":[69],"sketching":[70],"before":[71],"differentiation":[72],"biases":[73],"update":[75],"for":[76,170,189],"original":[78],"objective.":[79],"Model-Agnostic":[80],"Sparsified":[81],"instead":[83],"incorporates":[84],"sketch":[86],"distribution":[87],"into":[88],"its":[89,193],"which":[91],"makes":[92],"sketched":[93],"gradients":[94],"unbiased":[95],"admits":[97],"convergence":[98,138],"guarantees.":[99],"private":[101,137],"optimization,":[102],"adaptive":[106],"quantile":[107,117],"clipping":[108],"retains":[109,211],"bias":[110],"at":[111],"any":[112],"quantile,":[114],"while":[115],"suitable":[116],"schedules":[118],"recover":[119],"near-optimal":[120],"nonconvex":[121],"rates":[122],"under":[123,140],"heavy-tailed":[124],"noise.":[125],"We":[126,175],"then":[127],"combine":[128],"smoothed":[129],"normalization":[130],"with":[131,199],"error":[132],"compensation":[133],"obtain":[135],"distributed":[136],"guarantees":[139],"standard":[141],"without":[143],"assuming":[144],"bounded":[145],"gradients.":[146],"optimizers":[148],"based":[149],"linear":[151],"minimization":[152],"oracles,":[153],"develop":[155],"validate":[157],"generalized":[160],"smoothness":[161],"model":[162],"build":[164],"it":[166],"Gluon":[168],"framework":[169],"Muon-":[171],"Scion-style":[173],"methods.":[174],"also":[176],"orthogonalization":[181,187],"Muon":[183],"matters:":[184],"less":[185],"precise":[186],"calls":[188],"smaller":[190],"stepsizes,":[191],"so":[192],"precision":[194],"should":[195],"be":[196],"tuned":[197],"together":[198],"rate.":[202],"Overall,":[203],"are":[206],"better":[207],"understood":[208],"when":[209],"theory":[210],"features":[213],"determine":[215],"their":[216],"behavior.":[218]},"counts_by_year":[],"updated_date":"2026-10-31T12:09:11.096373","created_date":"2026-08-26T00:00:00"}