{"id":"https://openalex.org/W4414850286","doi":"https://doi.org/10.48550/arxiv.2502.17405","title":"Function-Space Learning Rates","display_name":"Function-Space Learning Rates","publication_year":2025,"publication_date":"2025-02-24","ids":{"openalex":"https://openalex.org/W4414850286","doi":"https://doi.org/10.48550/arxiv.2502.17405"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.17405","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.17405","pdf_url":"https://arxiv.org/pdf/2502.17405","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.17405","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056339931","display_name":"Edward Milsom","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Milsom, Edward","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028634767","display_name":"Ben Anson","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Anson, Ben","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5062539506","display_name":"Laurence Aitchison","orcid":"https://orcid.org/0000-0003-3681-4607","observed_orcids":["https://orcid.org/0000-0003-3681-4607"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aitchison, Laurence","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9276999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9276999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.024399999529123306,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.021199999377131462,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/neural-network-training-dynamics","display_name":"neural network training dynamics","score":0.7480000257492065},{"id":"https://openalex.org/keywords/hyperparameter-transfer","display_name":"hyperparameter transfer","score":0.8349999785423279},{"id":"https://openalex.org/keywords/model-scaling","display_name":"model scaling","score":0.5009999871253967},{"id":"https://openalex.org/keywords/lora","display_name":"LoRa","score":0.7739999890327454},{"id":"https://openalex.org/keywords/skip-connections","display_name":"skip connections","score":0.5529999732971191},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.7889999747276306}],"concepts":[{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.8026000261306763},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6621999740600586},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.598800003528595},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.5884000062942505},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5716000199317932},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.492000013589859},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.48579999804496765},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4684000015258789},{"id":"https://openalex.org/C10485038","wikidata":"https://www.wikidata.org/wiki/Q48996162","display_name":"Hyperparameter optimization","level":3,"score":0.42419999837875366},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4178999960422516},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4041999876499176},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.3662000000476837},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.36469998955726624},{"id":"https://openalex.org/C81299745","wikidata":"https://www.wikidata.org/wiki/Q334269","display_name":"Transfer function","level":2,"score":0.3635999858379364},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3504999876022339},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.3472000062465668},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.31679999828338623},{"id":"https://openalex.org/C83546350","wikidata":"https://www.wikidata.org/wiki/Q1139051","display_name":"Regression","level":2,"score":0.2766000032424927},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.27390000224113464},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2720000147819519},{"id":"https://openalex.org/C2779127903","wikidata":"https://www.wikidata.org/wiki/Q6510194","display_name":"Learning rule","level":3,"score":0.2712000012397766},{"id":"https://openalex.org/C91873725","wikidata":"https://www.wikidata.org/wiki/Q3445816","display_name":"Function approximation","level":3,"score":0.2703999876976013},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.25459998846054077},{"id":"https://openalex.org/C2776175482","wikidata":"https://www.wikidata.org/wiki/Q1195816","display_name":"Transfer (computing)","level":2,"score":0.25200000405311584}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.17405","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.17405","pdf_url":"https://arxiv.org/pdf/2502.17405","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.17405","doi":"https://doi.org/10.48550/arxiv.2502.17405","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.17405","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.17405","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.17405","pdf_url":"https://arxiv.org/pdf/2502.17405","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320320360","display_name":"University of Bristol","ror":"https://ror.org/0524sp257"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4414850286.pdf","grobid_xml":"https://content.openalex.org/works/W4414850286.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W1980067858","https://openalex.org/W2030434042","https://openalex.org/W2155894447","https://openalex.org/W2889237180","https://openalex.org/W2963095610","https://openalex.org/W2964121744","https://openalex.org/W2980282514","https://openalex.org/W2988294134","https://openalex.org/W3088274487","https://openalex.org/W3094151208","https://openalex.org/W3168867926","https://openalex.org/W4286717651","https://openalex.org/W4320722432","https://openalex.org/W4387226075","https://openalex.org/W4388090547","https://openalex.org/W4399317953","https://openalex.org/W4400482212","https://openalex.org/W4401306886","https://openalex.org/W4415796879","https://openalex.org/W4415798023","https://openalex.org/W4415798467"],"related_works":["https://openalex.org/W7214768121","https://openalex.org/W4404602505","https://openalex.org/W2804613085","https://openalex.org/W7163145411","https://openalex.org/W6929437511","https://openalex.org/W3199750449","https://openalex.org/W7162077425","https://openalex.org/W1957223602","https://openalex.org/W4415344525","https://openalex.org/W7221570521"],"abstract_inverted_index":{"We":[0,43,81],"consider":[1],"layerwise":[2,135],"function-space":[3,51,121,145],"learning":[4,32,52,122,136],"rates,":[5,33],"which":[6,34],"measure":[7,48],"the":[8,11,36,74,88],"magnitude":[9,37],"of":[10,38,90],"change":[12],"in":[13,19,40,54,95,160],"a":[14,25,64,110,126],"neural":[15,56,92],"network's":[16],"output":[17],"function":[18,96],"response":[20],"to":[21,24,47,113,142],"an":[22],"update":[23],"parameter":[26,41,100],"tensor.":[27],"This":[28],"contrasts":[29],"with":[30,165,170],"traditional":[31],"describe":[35],"changes":[39],"space.":[42],"develop":[44],"efficient":[45],"methods":[46],"and":[49,102,157,168],"set":[50],"rates":[53,123,137],"arbitrary":[55],"networks,":[57],"requiring":[58],"only":[59],"minimal":[60],"computational":[61],"overhead":[62],"through":[63],"few":[65],"additional":[66],"backward":[67],"passes":[68],"that":[69],"can":[70],"be":[71],"performed":[72],"at":[73],"start":[75],"of,":[76],"or":[77],"periodically":[78],"during,":[79],"training.":[80],"demonstrate":[82],"two":[83],"key":[84],"applications:":[85],"(1)":[86],"analysing":[87],"dynamics":[89],"standard":[91],"network":[93],"optimisers":[94],"space,":[97,101],"rather":[98],"than":[99],"(2)":[103],"introducing":[104],"FLeRM":[105,119,147],"(Function-space":[106],"Learning":[107],"Rate":[108],"Matching),":[109],"novel":[111],"approach":[112],"hyperparameter":[114,149],"transfer":[115,150],"across":[116,151],"model":[117,152],"scales.":[118],"records":[120],"while":[124],"training":[125,139],"small,":[127],"cheap":[128],"base":[129],"model,":[130],"then":[131],"automatically":[132],"adjusts":[133],"parameter-space":[134],"when":[138],"larger":[140],"models":[141],"maintain":[143],"consistent":[144],"updates.":[146],"gives":[148],"width,":[153],"depth,":[154],"initialisation":[155],"scale,":[156],"LoRA":[158],"rank":[159],"various":[161],"architectures":[162],"including":[163],"MLPs":[164],"residual":[166],"connections":[167],"transformers":[169],"different":[171],"layer":[172],"normalisation":[173],"schemes.":[174]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2025-10-10T00:00:00"}