{"id":"https://openalex.org/W7134913291","doi":"https://doi.org/10.48550/arxiv.2603.09950","title":"When Learning Rates Go Wrong: Early Structural Signals in PPO Actor-Critic","display_name":"When Learning Rates Go Wrong: Early Structural Signals in PPO Actor-Critic","publication_year":2026,"publication_date":"2026-03-10","ids":{"openalex":"https://openalex.org/W7134913291","doi":"https://doi.org/10.48550/arxiv.2603.09950"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.09950","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09950","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.09950","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128780925","display_name":"Alberto Fern\u00e1ndez-Hern\u00e1ndez","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fern\u00e1ndez-Hern\u00e1ndez, Alberto","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124332705","display_name":"Cristian P\u00e9rez-Corral","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"P\u00e9rez-Corral, Cristian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007319120","display_name":"Jose I. Mestre","orcid":"https://orcid.org/0000-0001-5954-4131"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mestre, Jose I.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128789540","display_name":"Manuel F. Dolz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dolz, Manuel F.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014020713","display_name":"Jos\u00e9 Luis Duato","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Duato, Jose","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128744619","display_name":"Enrique S. Quintana-Ort\u00ed","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Quintana-Ort\u00ed, Enrique S.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.40549999475479126,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.40549999475479126,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12611","display_name":"Neural Networks and Reservoir Computing","score":0.25099998712539673,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.06669999659061432,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5192000269889832},{"id":"https://openalex.org/keywords/hyperparameter","display_name":"Hyperparameter","score":0.4440000057220459},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.4392000138759613},{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.42179998755455017},{"id":"https://openalex.org/keywords/sign","display_name":"Sign (mathematics)","score":0.3190000057220459},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.3160000145435333},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.3107999861240387}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6754999756813049},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5192000269889832},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49540001153945923},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.4440000057220459},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.4392000138759613},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.42179998755455017},{"id":"https://openalex.org/C139676723","wikidata":"https://www.wikidata.org/wiki/Q1193832","display_name":"Sign (mathematics)","level":2,"score":0.3190000057220459},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.3160000145435333},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.31470000743865967},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.3107999861240387},{"id":"https://openalex.org/C50965678","wikidata":"https://www.wikidata.org/wiki/Q2724302","display_name":"Abnormality","level":2,"score":0.2793999910354614},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.2743000090122223},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.2712000012397766},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.2655999958515167},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.26409998536109924},{"id":"https://openalex.org/C31170391","wikidata":"https://www.wikidata.org/wiki/Q188619","display_name":"Hierarchy","level":2,"score":0.2612000107765198},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.25589999556541443},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.25049999356269836}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.09950","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09950","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.09950","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09950","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.6837539672851562,"display_name":"Reduced inequalities"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Deep":[0],"Reinforcement":[1],"Learning":[2],"systems":[3],"are":[4],"highly":[5],"sensitive":[6],"to":[7,35],"the":[8,52,55,59,62,70,106,112,192,200,213],"learning":[9],"rate":[10],"(LR),":[11],"and":[12,15,89,96,120,178],"selecting":[13],"stable":[14],"performant":[16],"training":[17,132],"runs":[18,225],"often":[19],"requires":[20],"extensive":[21],"hyperparameter":[22],"search.":[23],"In":[24,187],"Proximal":[25],"Policy":[26],"Optimization":[27],"(PPO)":[28],"actor--critic":[29],"methods,":[30],"small":[31],"LR":[32,40,95,136],"values":[33,41],"lead":[34],"slow":[36],"convergence,":[37],"whereas":[38,156],"large":[39],"may":[42],"induce":[43],"instability":[44],"or":[45],"collapse.":[46],"We":[47,81,138,167],"analyse":[48],"this":[49,188],"phenomenon":[50],"from":[51],"behavior":[53],"of":[54,72,87,105,131,223],"hidden":[56],"neurons":[57],"in":[58,149,216],"network":[60],"using":[61],"Overfitting-Underfitting":[63],"Indicator":[64],"(OUI),":[65],"a":[66,77,91,102,140],"metric":[67],"that":[68,125],"quantifies":[69],"balance":[71],"binary":[73],"activation":[74,97],"patterns":[75],"over":[76,184],"fixed":[78],"probe":[79],"batch.":[80],"introduce":[82],"an":[83,150],"efficient":[84],"batch-based":[85],"formulation":[86],"OUI":[88,126,152,165,190,197,211],"derive":[90],"theoretical":[92],"connection":[93],"between":[94,135],"sign":[98],"changes,":[99],"clarifying":[100],"how":[101],"correct":[103],"evolution":[104],"neuron's":[107],"inner":[108],"structure":[109],"depends":[110],"on":[111],"step":[113],"size.":[114],"Empirically,":[115],"across":[116],"three":[117],"discrete-control":[118],"environments":[119],"multiple":[121],"seeds,":[122],"we":[123],"show":[124],"measured":[127],"at":[128,203],"only":[129],"10\\%":[130],"already":[133],"discriminates":[134],"regimes.":[137],"observe":[139],"consistent":[141],"asymmetry:":[142],"critic":[143],"networks":[144,158],"achieving":[145,159],"highest":[146,160,214],"return":[147,161,209],"operate":[148],"intermediate":[151],"band":[153],"(avoiding":[154],"saturation),":[155],"actor":[157],"exhibit":[162],"comparatively":[163],"high":[164],"values.":[166],"then":[168],"compare":[169],"OUI-based":[170],"screening":[171,195,218],"rules":[172],"against":[173],"early":[174,194,208],"return,":[175],"clip-based,":[176],"divergence-based,":[177],"flip-based":[179],"criteria":[180],"under":[181],"matched":[182],"recall":[183],"successful":[185],"runs.":[186],"setting,":[189],"provides":[191],"strongest":[193],"signal:":[196],"alone":[198],"achieves":[199],"best":[201],"precision":[202,215],"broader":[204],"recall,":[205],"while":[206],"combining":[207],"with":[210],"yields":[212],"best-performing":[217],"regimes,":[219],"enabling":[220],"aggressive":[221],"pruning":[222],"unpromising":[224],"without":[226],"requiring":[227],"full":[228],"training.":[229]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-12T00:00:00"}