{"id":"https://openalex.org/W7166877317","doi":"https://doi.org/10.18653/v1/2026.findings-acl.1861","title":"How to Set the Learning Rate for Large-Scale Pre-training?","display_name":"How to Set the Learning Rate for Large-Scale Pre-training?","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166877317","doi":"https://doi.org/10.18653/v1/2026.findings-acl.1861"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.findings-acl.1861","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1861","pdf_url":"https://aclanthology.org/2026.findings-acl.1861.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.findings-acl.1861.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139743199","display_name":"Yunhua Zhou","orcid":null},"institutions":[{"id":"https://openalex.org/I4210091156","display_name":"SAIC-GM (China)","ror":"https://ror.org/00h64t852","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210091156"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yunhua Zhou","raw_affiliation_strings":["Shanghai AI Laboratory , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory , China","institution_ids":["https://openalex.org/I4210091156"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139757325","display_name":"Shuhao Xing","orcid":null},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]},{"id":"https://openalex.org/I4210091156","display_name":"SAIC-GM (China)","ror":"https://ror.org/00h64t852","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210091156"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuhao Xing","raw_affiliation_strings":["Fudan University , China","Shanghai AI Laboratory , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fudan University , China","institution_ids":["https://openalex.org/I24943067"]},{"raw_affiliation_string":"Shanghai AI Laboratory , China","institution_ids":["https://openalex.org/I4210091156"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139729723","display_name":"Junhao Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4210091156","display_name":"SAIC-GM (China)","ror":"https://ror.org/00h64t852","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210091156"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junhao Huang","raw_affiliation_strings":["Shanghai AI Laboratory , China","Shanghai JiaoTong University , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory , China","institution_ids":["https://openalex.org/I4210091156"]},{"raw_affiliation_string":"Shanghai JiaoTong University , China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139804213","display_name":"Xipeng Qiu","orcid":null},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xipeng Qiu","raw_affiliation_strings":["Fudan University , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fudan University , China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5139747848","display_name":"Qipeng Guo","orcid":null},"institutions":[{"id":"https://openalex.org/I4210091156","display_name":"SAIC-GM (China)","ror":"https://ror.org/00h64t852","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210091156"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qipeng Guo","raw_affiliation_strings":["Shanghai AI Laboratory , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory , China","institution_ids":["https://openalex.org/I4210091156"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.96443093,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"37344","last_page":"37361"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.05290000140666962,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.05290000140666962,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13844","display_name":"Higher Education Learning Practices","score":0.030500000342726707,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.028300000354647636,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5289000272750854},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.3450999855995178},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.2522999942302704},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.2337999939918518}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5465999841690063},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5289000272750854},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47859999537467957},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3450999855995178},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.29089999198913574},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.267300009727478},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2522999942302704},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.23680000007152557},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.2337999939918518},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.225600004196167}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.findings-acl.1861","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1861","pdf_url":"https://aclanthology.org/2026.findings-acl.1861.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.findings-acl.1861","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1861","pdf_url":"https://aclanthology.org/2026.findings-acl.1861.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166877317.pdf","grobid_xml":"https://content.openalex.org/works/W7166877317.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Optimal":[0],"configuration":[1],"of":[2,94,99,118,124,140,158],"the":[3,16,25,30,54,68,87,92,97,116,138,141,155,166],"learning":[4,163],"rate":[5],"(LR)":[6],"is":[7,28],"a":[8,60,128,151,183],"fundamental":[9],"yet":[10],"formidable":[11],"challenge":[12,137],"in":[13,122,145,174],"large-scale":[14,146,175],"pre-training.Given":[15],"stringent":[17],"trade-off":[18],"between":[19,131],"training":[20,159],"costs":[21],"and":[22,52,112,161,182],"model":[23,108],"performance,":[24],"pivotal":[26],"question":[27],"whether":[29],"optimal":[31],"LR":[32],"can":[33],"be":[34],"accurately":[35],"extrapolated":[36],"from":[37,71],"low-cost":[38],"experiments.In":[39],"this":[40,44],"paper,":[41],"we":[42,57,90,126,149],"formalize":[43],"investigation":[45],"into":[46],"two":[47,133],"distinct":[48],"research":[49,121],"paradigms:":[50],"Fitting":[51,55],"Transfer.Within":[53],"Paradigm,":[56,89],"innovatively":[58],"introduce":[59],"Scaling":[61],"Law":[62],"for":[63,187],"search":[64,69],"factor,":[65],"effectively":[66],"reducing":[67],"complexity":[70],"O(n":[72,76],"3":[73],")":[74,83],"to":[75,96,106,164],"\u2022":[77,80],"C":[78,81],"D":[79],"\u03b7":[82],"via":[84],"predictive":[85],"modeling.Within":[86],"Transfer":[88],"extend":[91],"principles":[93],"\u00b5Transfer":[95,144],"Mixture":[98],"Experts":[100],"(MoE)":[101],"architecture,":[102],"broadening":[103],"its":[104],"applicability":[105],"encompass":[107],"depth,":[109],"weight":[110],"decay,":[111],"token":[113],"horizons.By":[114],"pushing":[115],"boundaries":[117],"existing":[119],"hyperparameter":[120],"terms":[123],"scale,":[125],"conduct":[127],"comprehensive":[129],"comparison":[130],"these":[132],"paradigms.Our":[134],"empirical":[135],"results":[136],"scalability":[139],"widely":[142],"adopted":[143],"pre-training":[147],"scenarios.Furthermore,":[148],"provide":[150],"rigorous":[152],"analysis":[153],"through":[154],"dual":[156],"lenses":[157],"stability":[160],"feature":[162],"elucidate":[165],"underlying":[167],"reasons":[168],"why":[169],"module-wise":[170],"parameter":[171],"tuning":[172],"underperforms":[173],"settings.This":[176],"work":[177],"offers":[178],"systematic":[179],"practical":[180],"guidelines":[181],"fresh":[184],"theoretical":[185],"perspective":[186],"optimizing":[188],"industriallevel":[189],"pre-training.":[190]},"counts_by_year":[],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2026-07-02T00:00:00"}