{"id":"https://openalex.org/W7220764435","doi":null,"title":"Understanding the mechanism of pretraining stabilization heuristics: A variance-oriented perspective","display_name":"Understanding the mechanism of pretraining stabilization heuristics: A variance-oriented perspective","publication_year":2024,"publication_date":"2024-04-08","ids":{"openalex":"https://openalex.org/W7220764435"},"language":"en","primary_location":{"id":"pmh:oai:www.ideals.illinois.edu:2142/124173","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/124173","pdf_url":"https://www.ideals.illinois.edu/items/131273/bitstreams/436411/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.ideals.illinois.edu/items/131273/bitstreams/436411/data.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5157783956","display_name":"Liyuan Liu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Liu, Liyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5157783956"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7109000086784363,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7109000086784363,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.2240999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.020899999886751175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/language-model-pretraining","display_name":"language model pretraining","score":0.9789999723434448},{"id":"https://openalex.org/keywords/adaptive-learning-rate","display_name":"adaptive learning rate","score":0.7570000290870667},{"id":"https://openalex.org/keywords/model-initialization","display_name":"model initialization","score":0.7639999985694885},{"id":"https://openalex.org/keywords/gradient-approximation","display_name":"gradient approximation","score":0.6499999761581421},{"id":"https://openalex.org/keywords/training-stability","display_name":"training stability","score":0.41600000858306885},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.5270000100135803},{"id":"https://openalex.org/keywords/artificial-general-intelligence","display_name":"artificial general intelligence","score":0.5580000281333923}],"concepts":[{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.7609000205993652},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.6401000022888184},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.39879998564720154},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.39559999108314514},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3540000021457672},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.33799999952316284},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.313400000333786},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.28700000047683716},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2842000126838684},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.27309998869895935},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.25279998779296875}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:www.ideals.illinois.edu:2142/124173","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/124173","pdf_url":"https://www.ideals.illinois.edu/items/131273/bitstreams/436411/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:www.ideals.illinois.edu:2142/124230","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/124230","pdf_url":"https://www.ideals.illinois.edu/items/131330/bitstreams/436568/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:www.ideals.illinois.edu:2142/124173","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/124173","pdf_url":"https://www.ideals.illinois.edu/items/131273/bitstreams/436411/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220764435.pdf","grobid_xml":"https://content.openalex.org/works/W7220764435.grobid-xml"},"referenced_works_count":36,"referenced_works":["https://openalex.org/W6908809","https://openalex.org/W1821462560","https://openalex.org/W1959608418","https://openalex.org/W2125930537","https://openalex.org/W2242818861","https://openalex.org/W2259472270","https://openalex.org/W2622263826","https://openalex.org/W2750384547","https://openalex.org/W2796705611","https://openalex.org/W2899663614","https://openalex.org/W2965373594","https://openalex.org/W2981852735","https://openalex.org/W2989571009","https://openalex.org/W2994673210","https://openalex.org/W2995015695","https://openalex.org/W2997347790","https://openalex.org/W3010768098","https://openalex.org/W3030163527","https://openalex.org/W3033187248","https://openalex.org/W3034995656","https://openalex.org/W3037932933","https://openalex.org/W3040573126","https://openalex.org/W3066373881","https://openalex.org/W3098608318","https://openalex.org/W3119866685","https://openalex.org/W3171649327","https://openalex.org/W4224088803","https://openalex.org/W4224308101","https://openalex.org/W4282972970","https://openalex.org/W4327810158","https://openalex.org/W4360836968","https://openalex.org/W4362515116","https://openalex.org/W4384918448","https://openalex.org/W7133219378","https://openalex.org/W7133224204","https://openalex.org/W7133227004"],"related_works":["https://openalex.org/W7220773204","https://openalex.org/W7220798479","https://openalex.org/W2479247647","https://openalex.org/W6922276705","https://openalex.org/W7220815301","https://openalex.org/W7042719468","https://openalex.org/W7163056241","https://openalex.org/W7220755006","https://openalex.org/W7220816180","https://openalex.org/W7168350485"],"abstract_inverted_index":{"Language":[0],"model":[1,132,171,180,260,564],"pretraining":[2,68,96,126,143,249,289],"has":[3,16,295,304,571],"been":[4,17,296,323,370,408,572],"breaking":[5],"the":[6,22,74,91,116,122,164,190,194,207,211,229,234,248,268,271,326,341,389,398,420,426,433,524,601,624,636,643,648],"glass":[7],"ceiling":[8],"for":[9,160,253,258,265,288,291,353,436,499,592,603],"various":[10],"natural":[11],"language":[12,507],"processing":[13],"tasks":[14],"and":[15,35,40,49,103,134,170,179,231,263,285,300,308,316,329,359,367,376,391,413,424,454,505,528,544,584,607],"viewed":[18],"as":[19,59,183,283],"one":[20],"of":[21,26,33,47,61,66,77,124,166,210,233,270,310,313,331,374,512,588,652],"most":[23],"significant":[24,476,577],"successes":[25],"deep":[27,503],"learning,":[28],"continuously":[29],"challenging":[30,299,482],"our":[31,430,618],"understanding":[32],"learning":[34,129,168,176,254,504],"cognition.Recently":[36],"models,":[37],"including":[38,251],"GPT-4":[39],"BART,":[41],"fueled":[42],"by":[43,635],"an":[44],"unprecedented":[45,52],"scale":[46],"computing":[48],"data,":[50],"exhibit":[51],"intelligence,":[53],"that":[54,118,141,174,303,400,616],"some":[55],"even":[56],"refer":[57],"to":[58,89,99,121,188,227,246,334,347,386,393,422,447,472,493,501,539,547,556,629,640],"\"sparks":[60],"artificial":[62],"general":[63],"intelligence\".The":[64],"success":[65],"large-scale":[67],"hinges":[69],"on":[70,397,439,474,521],"intricate":[71],"engineering":[72],"heuristics.While":[73],"empirical":[75,223],"benefits":[76],"these":[78,95,142,238],"heuristics":[79,117,145],"are":[80,119,225],"evident,":[81],"their":[82,101],"underlying":[83,94,208,644],"mechanisms":[84,102],"remain":[85],"elusive.This":[86],"dissertation":[87,241,449],"endeavors":[88],"demystify":[90],"mathematical":[92],"principles":[93],"heuristics,":[97],"aiming":[98],"illuminate":[100],"potentially":[104],"guide":[105],"future":[106],"algorithm":[107],"developments.Adopting":[108],"a":[109,151,157,201,297,371,518,541,562,576],"variance-oriented":[110],"perspective,":[111],"my":[112,338,343,348,363,380,395,444,448,459,467,581,593,604,609,653],"research":[113,569,582,605],"rigorously":[114],"inspects":[115],"pivotal":[120],"stability":[123],"current":[125],"practices,":[127,250],"emphasizing":[128,432],"rate":[130,169,177,255],"warmup,":[131,256],"initialization,":[133,261],"gradient":[135,197,266],"approximation.In":[136],"this":[137,240,293],"dissertation,":[138],"I":[139,162,186,205,336,401],"show":[140],"stabilization":[144],"can":[146],"be":[147],"coherently":[148],"elucidated":[149],"with":[150,217,485,532],"unified":[152],"framework":[153],"anchored":[154],"in":[155,193,418,429,466,568,579,611],"variance,":[156],"classical":[158],"metric":[159],"stability.First,":[161],"analyze":[163],"variance":[165,184],"adaptive":[167],"outputs,":[172],"revealing":[173],"both":[175],"warmup":[178],"initialization":[181],"function":[182],"modulators.Then,":[185],"move":[187],"explore":[189,502,642],"variance-bias":[191],"tradeoff":[192],"discrete":[195],"variable":[196],"approximation,":[198],"i.e.,":[199],"employing":[200],"numerical":[202],"ODE":[203],"framework,":[204],"unveil":[206],"dynamics":[209],"approximation":[212],"bias,":[213],"achieving":[214],"secondorder":[215],"precision":[216],"minimal":[218,278],"computational":[219],"overhead.Besides":[220],"theoretical":[221],"results,":[222],"verifications":[224],"conducted":[226],"verify":[228],"assumptions":[230],"applicability":[232],"recognized":[235,272],"principles.Building":[236],"upon":[237],"insights,":[239],"introduces":[242],"novel":[243],"techniques":[244],"designed":[245],"advance":[247],"RAdam":[252],"Admin":[257],"Transformer":[259],"ReinMax":[262],"SparseMixer":[264],"approximation.Under":[267],"guidance":[269,567,646],"principles,":[273],"all":[274],"proposed":[275,619],"methods":[276],"require":[277],"trial-and-error":[279],"configurations,":[280],"thereby":[281],"emerging":[282],"robust":[284],"high-perform":[286],"tools":[287],"practices":[290],"adaptations.Completing":[292],"Ph.D.":[294],"profoundly":[298],"rewarding":[301],"journey":[302],"spanned":[305],"several":[306],"years":[307],"consisted":[309],"countless":[311],"hours":[312],"hard":[314],"work":[315],"dedication.This":[317],"monumental":[318],"task":[319],"would":[320,405,554],"not":[321,406,630],"have":[322,369,377,387,407,456],"possible":[324,409,498],"without":[325,410],"support,":[327,356],"guidance,":[328],"patience":[330],"many":[332],"individuals":[333],"whom":[335],"owe":[337],"sincere":[339,445],"gratitude.At":[340],"forefront,":[342],"deepest":[344],"appreciation":[345],"goes":[346],"advisor,":[349,595],"Dr.":[350,415,533,559,596],"Jiawei":[351],"Han,":[352],"his":[354],"unwavering":[355],"invaluable":[357],"advice,":[358],"insightful":[360,566],"feedback":[361],"throughout":[362],"research.Dr.":[364],"Han's":[365],"brilliance":[366],"passion":[368],"constant":[372],"source":[373],"inspiration":[375],"significantly":[378,457],"shaped":[379],"growth.I":[381],"always":[382],"consider":[383],"myself":[384],"fortunate":[385],"had":[388],"opportunity":[390],"flexibility":[392],"spend":[394],"Ph.D.working":[396],"problems":[399,477],"believed":[402],"in,":[403],"which":[404],"Dr.Han's":[411],"support":[412],"guidance.Moreover,":[414],"Han":[416],"instilled":[417],"me":[419,471,491,500,538,628],"wisdom":[421],"identify":[423],"address":[425],"core":[427],"issues":[428],"field,":[431],"real":[434,437,440],"solution":[435],"problem":[438],"data.I":[441],"then":[442],"extend":[443],"thanks":[446,589],"committee":[450],"members,":[451],"whose":[452,565],"insights":[453],"suggestions":[455],"enriched":[458],"research.Prof.":[460],"Chengxiang":[461],"Zhai":[462],"provided":[463,490],"valuable":[464],"advice":[465],"second":[468],"year,":[469],"urging":[470],"focus":[473],"tackling":[475],"rather":[478],"than":[479],"solely":[480],"pursuing":[481],"ones.Moreover,":[483],"together":[484],"Prof.":[486,513],"Jian":[487],"Peng,":[488],"he":[489],"access":[492],"GPU":[494],"resources,":[495],"making":[496],"it":[497],"neural":[506],"modeling.I":[508],"am":[509],"also":[510,557],"appreciative":[511],"Heng":[514],"Ji,":[515],"who":[516,599],"offered":[517],"unique":[519],"perspective":[520],"research,":[522,548],"highlighting":[523],"balance":[525],"between":[526],"academic":[527,654],"industry.My":[529],"internship":[530],"experience":[531],"Matthew":[534],"E.":[535],"Peters":[536],"led":[537],"adopt":[540],"more":[542],"deliberate":[543],"quality-focused":[545],"approach":[546],"prioritizing":[549],"impactful":[550],"publications":[551],"over":[552],"quantity.I":[553],"like":[555],"acknowledge":[558],"Jianfeng":[560],"Gao,":[561],"role":[563,578],"direction":[570],"crucial.His":[573],"mentorship":[574],"played":[575],"shaping":[580],"philosophy":[583],"approach.A":[585],"special":[586],"note":[587],"is":[590],"reserved":[591],"undergraduate":[594],"Linli":[597],"Xu,":[598],"laid":[600],"foundation":[602],"career":[606,655],"kindled":[608],"interest":[610],"machine":[612],"learning.I":[613],"still":[614],"remember":[615],"when":[617],"community":[620],"detection":[621],"method":[622],"achieves":[623],"state-of-the-art,":[625],"she":[626],"encouraged":[627],"stop":[631],"at":[632],"being":[633],"satisfied":[634],"good":[637],"performance":[638],"but":[639],"continue":[641],"mechanism.Her":[645],"during":[647],"critical":[649],"initial":[650],"phase":[651],"was":[656],"instrumental.":[657]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-10-07T00:00:00"}