{"id":"https://openalex.org/W7161557412","doi":"https://doi.org/10.48550/arxiv.2605.16184","title":"Runtime-Orchestrated Second-Order Optimization for Scalable LLM Training","display_name":"Runtime-Orchestrated Second-Order Optimization for Scalable LLM Training","publication_year":2026,"publication_date":"2026-05-15","ids":{"openalex":"https://openalex.org/W7161557412","doi":"https://doi.org/10.48550/arxiv.2605.16184"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.16184","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.16184","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.16184","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136403938","display_name":"Yishun Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Yishun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021853581","display_name":"Junhao Zhang","orcid":"https://orcid.org/0009-0004-0990-672X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Junhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136373404","display_name":"Zeyu Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Zeyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129676934","display_name":"Wes Armour","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Armour, Wes","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.8130999803543091,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.8130999803543091,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10904","display_name":"Embedded Systems Design Techniques","score":0.03889999911189079,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.014399999752640724,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.6970999836921692},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6513000130653381},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.5465999841690063},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.4867999851703644},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.41940000653266907},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.41260001063346863},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.3919999897480011},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.3824000060558319},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.3779999911785126}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8676999807357788},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6970999836921692},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6513000130653381},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.5465999841690063},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5069000124931335},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4867999851703644},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.44530001282691956},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.41940000653266907},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.41260001063346863},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.3919999897480011},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.3824000060558319},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.3779999911785126},{"id":"https://openalex.org/C126831891","wikidata":"https://www.wikidata.org/wiki/Q221673","display_name":"Host (biology)","level":2,"score":0.36500000953674316},{"id":"https://openalex.org/C2778119891","wikidata":"https://www.wikidata.org/wiki/Q477690","display_name":"CUDA","level":2,"score":0.3490999937057495},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.34279999136924744},{"id":"https://openalex.org/C115874739","wikidata":"https://www.wikidata.org/wiki/Q825377","display_name":"Critical path method","level":2,"score":0.34200000762939453},{"id":"https://openalex.org/C193702766","wikidata":"https://www.wikidata.org/wiki/Q1414548","display_name":"Concurrency","level":2,"score":0.3073999881744385},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.30390000343322754},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.2939999997615814},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2935999929904938},{"id":"https://openalex.org/C201410400","wikidata":"https://www.wikidata.org/wiki/Q1064412","display_name":"Multithreading","level":3,"score":0.2930999994277954},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.29269999265670776},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.2865000069141388},{"id":"https://openalex.org/C21442007","wikidata":"https://www.wikidata.org/wiki/Q1027879","display_name":"Graphics","level":2,"score":0.28600001335144043},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.27410000562667847},{"id":"https://openalex.org/C86111242","wikidata":"https://www.wikidata.org/wiki/Q859595","display_name":"Coprocessor","level":2,"score":0.2711000144481659},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.26589998602867126},{"id":"https://openalex.org/C50630238","wikidata":"https://www.wikidata.org/wiki/Q971505","display_name":"General-purpose computing on graphics processing units","level":3,"score":0.2529999911785126},{"id":"https://openalex.org/C12269588","wikidata":"https://www.wikidata.org/wiki/Q132364","display_name":"Communications protocol","level":2,"score":0.2524999976158142},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.25200000405311584},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.2515000104904175}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.16184","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.16184","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.16184","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.16184","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Second-order":[0],"methods":[1],"offer":[2],"an":[3],"attractive":[4],"path":[5],"toward":[6],"more":[7],"sample-efficient":[8],"LLM":[9,196],"training,":[10,109],"but":[11,208],"their":[12],"practical":[13,201],"use":[14],"is":[15],"often":[16],"blocked":[17],"by":[18,41,203,209],"the":[19,47,59,101,179,205,222],"systems":[20],"cost":[21],"of":[22,182],"maintaining":[23],"and":[24,71,79,132,146,177,184,216],"updating":[25],"large":[26],"matrix-based":[27],"optimizer":[28,64,121,166,206,212],"states.":[29],"We":[30,126],"introduce":[31],"\\textbf{Asteria},":[32],"a":[33,112,137,142,155,187],"runtime":[34,80,223],"system":[35],"designed":[36],"to":[37,76,87,97],"remove":[38],"this":[39],"bottleneck":[40],"separating":[42],"second-order":[43,152,195],"optimization":[44,180],"logic":[45],"from":[46],"critical":[48],"GPU":[49,67,104,145],"training":[50,85,134,153,197],"path.":[51],"Rather":[52],"than":[53],"keeping":[54],"all":[55],"preconditioner":[56],"state":[57,65],"on":[58,100,129],"accelerator,":[60],"Asteria":[61,110,128,150],"dynamically":[62],"distributes":[63],"across":[66],"memory,":[68,70,149],"CPU":[69],"optional":[72],"NVMe":[73],"storage":[74],"according":[75],"architectural":[77],"constraints":[78],"pressure.":[81],"It":[82],"further":[83],"uses":[84],"hooks":[86],"prepare":[88],"shadow":[89],"states":[90],"in":[91,174,186],"advance,":[92],"allowing":[93],"expensive":[94],"inverse-root":[95],"computations":[96],"proceed":[98],"asynchronously":[99],"host":[102],"while":[103,119],"computation":[105],"continues.":[106],"For":[107],"distributed":[108,133,217],"employs":[111],"bounded-staleness":[113],"protocol":[114],"that":[115,194],"limits":[116],"synchronization":[117,218],"frequency":[118],"preserving":[120],"effectiveness":[122],"through":[123],"topology-aware":[124],"coordination.":[125],"evaluate":[127],"both":[130],"memory-constrained":[131],"settings.":[135],"On":[136,159],"DGX":[138],"Spark":[139],"platform":[140],"with":[141],"single":[143],"GB10":[144],"128GB":[147],"unified":[148],"supports":[151],"for":[154],"1B-parameter":[156],"language":[157,189],"model.":[158,190],"multi-node":[160],"GH200":[161],"systems,":[162],"it":[163],"lowers":[164],"visible":[165],"overhead,":[167],"reduces":[168],"recurring":[169],"latency":[170],"spikes,":[171],"accelerates":[172],"convergence":[173],"wall-clock":[175],"time,":[176],"maintains":[178],"advantages":[181],"SOAP":[183],"KL-Shampoo":[185],"7B-parameter":[188],"Our":[191],"results":[192],"suggest":[193],"can":[198],"be":[199],"made":[200],"not":[202],"simplifying":[204],"alone,":[207],"rethinking":[210],"how":[211],"state,":[213],"background":[214],"computation,":[215],"are":[219],"managed":[220],"at":[221],"level.":[224]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-19T00:00:00"}