{"id":"https://openalex.org/W7214707192","doi":"https://doi.org/10.48550/arxiv.2609.34667","title":"Two-Timescale Fine-tuning Provably Learns New Features for Two-Layer ReLU Networks","display_name":"Two-Timescale Fine-tuning Provably Learns New Features for Two-Layer ReLU Networks","publication_year":2026,"publication_date":"2026-09-28","ids":{"openalex":"https://openalex.org/W7214707192","doi":"https://doi.org/10.48550/arxiv.2609.34667"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.34667","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.34667","pdf_url":"https://arxiv.org/pdf/2609.34667","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.34667","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064037732","display_name":"\u00c9tienne Boursier","orcid":"https://orcid.org/0000-0002-7575-8575","observed_orcids":["https://orcid.org/0000-0002-7575-8575"]},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I277688954","display_name":"Universit\u00e9 Paris-Saclay","ror":"https://ror.org/03xjwb503","country_code":"FR","type":"education","lineage":["https://openalex.org/I277688954"]},{"id":"https://openalex.org/I3019441195","display_name":"Laboratoire de Math\u00e9matiques d'Orsay","ror":"https://ror.org/03ab0zs98","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I277688954","https://openalex.org/I3019441195"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Boursier, Etienne","raw_affiliation_strings":["INRIA , LMO , Universit\u00e9 Paris-Saclay , Orsay , France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"INRIA , LMO , Universit\u00e9 Paris-Saclay , Orsay , France","institution_ids":["https://openalex.org/I1326498283","https://openalex.org/I277688954","https://openalex.org/I3019441195"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061093552","display_name":"Nicolas Flammarion","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Flammarion, Nicolas","raw_affiliation_strings":["EPFL , Lausanne , Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"EPFL , Lausanne , Switzerland","institution_ids":["https://openalex.org/I5124864"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.7724999785423279,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.7724999785423279,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.16699999570846558,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.040699999779462814,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.9350000023841858},{"id":"https://openalex.org/keywords/pretraining","display_name":"pre-training","score":0.628000020980835},{"id":"https://openalex.org/keywords/representation-learning","display_name":"representation learning","score":0.5299999713897705},{"id":"https://openalex.org/keywords/implicit-bias","display_name":"implicit bias","score":0.7620000243186951},{"id":"https://openalex.org/keywords/random-initialization","display_name":"random initialization","score":0.7680000066757202},{"id":"https://openalex.org/keywords/teacher-student-models","display_name":"teacher-student models","score":0.45100000500679016},{"id":"https://openalex.org/keywords/data-scarcity","display_name":"data scarcity","score":0.40400001406669617},{"id":"https://openalex.org/keywords/parameter-recovery","display_name":"parameter recovery","score":0.3400000035762787}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7189000248908997},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.6858999729156494},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.651199996471405},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.42170000076293945},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39559999108314514},{"id":"https://openalex.org/C2982736386","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Statistical learning","level":2,"score":0.39149999618530273},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3659000098705292},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3569999933242798},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3531999886035919},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.35089999437332153},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.33570000529289246},{"id":"https://openalex.org/C83665646","wikidata":"https://www.wikidata.org/wiki/Q42139305","display_name":"Feature vector","level":2,"score":0.32760000228881836},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3190999925136566},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.30970001220703125},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.30399999022483826},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.2976999878883362},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.2955000102519989},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.2913999855518341},{"id":"https://openalex.org/C2781170535","wikidata":"https://www.wikidata.org/wiki/Q30587856","display_name":"Noisy data","level":2,"score":0.27399998903274536},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.27129998803138733},{"id":"https://openalex.org/C61326573","wikidata":"https://www.wikidata.org/wiki/Q1496376","display_name":"Gaussian process","level":3,"score":0.26460000872612},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.2574999928474426},{"id":"https://openalex.org/C176248197","wikidata":"https://www.wikidata.org/wiki/Q458526","display_name":"Probably approximately correct learning","level":4,"score":0.2524000108242035}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.34667","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.34667","pdf_url":"https://arxiv.org/pdf/2609.34667","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.34667","doi":"https://doi.org/10.48550/arxiv.2609.34667","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.34667","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.34667","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.34667","pdf_url":"https://arxiv.org/pdf/2609.34667","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G6902248818","display_name":"Training Overparametrized Models with SGD: the Impact of Noise","funder_award_id":"212111","funder_id":"https://openalex.org/F4320320924","funder_display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung"}],"funders":[{"id":"https://openalex.org/F4320320924","display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","ror":"https://ror.org/00yjd3n13"},{"id":"https://openalex.org/F4320324478","display_name":"Fondation Math\u00e9matique Jacques Hadamard","ror":"https://ror.org/0153me927"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214707192.pdf","grobid_xml":"https://content.openalex.org/works/W7214707192.grobid-xml"},"referenced_works_count":3,"referenced_works":["https://openalex.org/W2010353172","https://openalex.org/W4404986416","https://openalex.org/W4416380220"],"related_works":["https://openalex.org/W7131417496","https://openalex.org/W7214780597","https://openalex.org/W4415795458","https://openalex.org/W4404879567","https://openalex.org/W7112397960","https://openalex.org/W7127606758","https://openalex.org/W6929492077","https://openalex.org/W4415346780","https://openalex.org/W3005642076","https://openalex.org/W4401307234"],"abstract_inverted_index":{"Fine-tuning":[0],"pre-trained":[1,35,88,109],"models":[2],"on":[3],"specialized":[4],"tasks":[5],"with":[6,113,139],"scarce":[7,151],"data":[8],"is":[9,121],"central":[10],"to":[11,31,123],"modern":[12],"deep":[13],"learning.":[14],"Despite":[15],"its":[16],"empirical":[17],"success,":[18],"theoretical":[19],"understanding":[20],"of":[21,45,53,105,108,119],"fine-tuning":[22,33,97,152],"remains":[23],"limited.":[24],"We":[25],"introduce":[26],"a":[27,140],"Gaussian":[28],"multi-index":[29],"setting":[30],"study":[32],"from":[34,150],"weights,":[36],"where":[37],"the":[38,71,77,81,87,91,106,116,125],"teacher":[39],"network":[40],"has":[41],"$m+1$":[42],"features,":[43],"$m$":[44],"which":[46,54],"are":[47,99],"learned":[48,57],"during":[49,58],"pre-training":[50,133],"and":[51],"one":[52],"must":[55],"be":[56],"fine-tuning.":[59],"For":[60],"two-layer":[61],"ReLU":[62],"networks,":[63],"we":[64],"show":[65],"that":[66,132],"two-timescale":[67],"training,":[68],"i.e.,":[69],"updating":[70],"outer":[72],"weights":[73],"infinitely":[74],"faster":[75],"than":[76],"hidden":[78],"ones,":[79],"learns":[80],"new":[82],"task-specific":[83],"feature":[84,148],"while":[85],"preserving":[86],"ones":[89],"in":[90],"model":[92],"representation.":[93],"Moreover,":[94],"only":[95],"$\\mathcal{O}(d)$":[96],"samples":[98,120],"required":[100],"for":[101],"this":[102],"recovery,":[103],"independently":[104],"number":[107,118],"features.":[110],"In":[111],"contrast,":[112],"random":[114,145],"initialization,":[115,146],"same":[117],"insufficient":[122],"recover":[124],"target":[126],"parameters.":[127],"Our":[128],"results":[129],"therefore":[130],"demonstrate":[131],"can":[134],"induce":[135],"an":[136],"implicit":[137],"bias":[138],"clear":[141],"statistical":[142],"advantage":[143],"over":[144],"enabling":[147],"learning":[149],"data.":[153]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-30T00:00:00"}