{"id":"https://openalex.org/W7160950412","doi":"https://doi.org/10.48550/arxiv.2605.09212","title":"Rethinking Ratio-Based Trust Regions for Policy Optimization in Multi-Agent Reinforcement Learning","display_name":"Rethinking Ratio-Based Trust Regions for Policy Optimization in Multi-Agent Reinforcement Learning","publication_year":2026,"publication_date":"2026-05-09","ids":{"openalex":"https://openalex.org/W7160950412","doi":"https://doi.org/10.48550/arxiv.2605.09212"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.09212","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09212","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.09212","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051725666","display_name":"Chulabhaya Wijesundara","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wijesundara, Chulabhaya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033275466","display_name":"Andrea Baisero","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Baisero, Andrea","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135939962","display_name":"Zhongheng Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Zhongheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123140104","display_name":"Gregory Casta\u00f1\u00f3n","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Casta\u00f1\u00f3n, Gregory","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135997636","display_name":"Alan Carlin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Carlin, Alan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135976917","display_name":"Christopher Amato","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Amato, Christopher","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8159999847412109,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8159999847412109,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.012299999594688416,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.012000000104308128,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6345999836921692},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.5680000185966492},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.5253000259399414},{"id":"https://openalex.org/keywords/outlier","display_name":"Outlier","score":0.5012999773025513},{"id":"https://openalex.org/keywords/quadratic-equation","display_name":"Quadratic equation","score":0.49070000648498535},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.4537999927997589},{"id":"https://openalex.org/keywords/grammaticality","display_name":"Grammaticality","score":0.42980000376701355}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6345999836921692},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6139000058174133},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5680000185966492},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.5253000259399414},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.5041999816894531},{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.5012999773025513},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.49070000648498535},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.4537999927997589},{"id":"https://openalex.org/C2779525943","wikidata":"https://www.wikidata.org/wiki/Q1187300","display_name":"Grammaticality","level":3,"score":0.42980000376701355},{"id":"https://openalex.org/C83260615","wikidata":"https://www.wikidata.org/wiki/Q6773121","display_name":"Mars Exploration Program","level":2,"score":0.42309999465942383},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.3903999924659729},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.3458000123500824},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.32339999079704285},{"id":"https://openalex.org/C81845259","wikidata":"https://www.wikidata.org/wiki/Q290117","display_name":"Quadratic programming","level":2,"score":0.3003999888896942},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.29679998755455017},{"id":"https://openalex.org/C89109886","wikidata":"https://www.wikidata.org/wiki/Q1535924","display_name":"Trust region","level":3,"score":0.2718000113964081},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.2667999863624573},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2621000111103058},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.25270000100135803}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.09212","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09212","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.09212","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09212","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"score":0.7017087936401367,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Centralized":[0],"training":[1],"with":[2,122],"decentralized":[3,45],"execution":[4],"(CTDE)":[5],"is":[6],"a":[7,110,123],"standard":[8],"framework":[9],"for":[10,86],"cooperative":[11],"multi-agent":[12,146],"policy-gradient":[13],"reinforcement":[14],"learning,":[15],"allowing":[16],"agents":[17],"to":[18],"learn":[19],"from":[20,25,92,172,181],"joint":[21,53],"information":[22],"while":[23,95,132],"acting":[24],"local":[26,72],"observations.":[27],"Ratio-based":[28],"trust-region":[29,120,183],"methods":[30],"such":[31],"as":[32,136],"Multi-Agent":[33,39,106],"Proximal":[34],"Policy":[35,41],"Optimization":[36,42],"(MAPPO)":[37],"and":[38,89,153,160],"Simple":[40],"(MASPO)":[43],"update":[44],"actors":[46],"using":[47],"per-agent":[48],"probability":[49,102,137],"ratios":[50,138],"weighted":[51],"by":[52],"advantage":[54],"estimates.":[55],"Teammate":[56],"non-stationarity":[57],"increases":[58,67],"the":[59,68,71,173,176],"variance":[60,69],"of":[61,175],"these":[62,117,169],"advantages,":[63],"which":[64],"in":[65,70,162],"turn":[66],"ratio":[73],"updates.":[74],"This":[75],"exposes":[76],"two":[77],"method-specific":[78],"failure":[79],"modes:":[80],"MAPPO's":[81],"additive":[82,118],"clipping":[83],"removes":[84],"gradients":[85,131],"outlier":[87],"samples":[88],"weakens":[90],"recovery":[91],"policy":[93,112],"drift,":[94],"MASPO's":[96],"soft":[97],"quadratic":[98],"penalty":[99],"can":[100],"allow":[101],"collapse.":[103],"We":[104],"introduce":[105],"Ratio":[107],"Symmetry":[108],"(MARS),":[109],"novel":[111,149],"optimization":[113],"objective":[114],"that":[115,168],"replaces":[116],"ratio-based":[119],"mechanisms":[121],"multiplicatively":[124],"symmetric":[125,177],"geometric":[126],"barrier.":[127],"MARS":[128,155],"preserves":[129],"corrective":[130],"assigning":[133],"unbounded":[134],"cost":[135],"approach":[139],"zero.":[140],"Across":[141],"47":[142],"tasks":[143],"spanning":[144],"eight":[145],"environments,":[147],"including":[148],"JAX":[150],"benchmarks":[151],"PaxMen":[152],"AeroJAX,":[154],"matches":[156],"or":[157],"exceeds":[158],"MAPPO":[159],"MASPO":[161],"aggregate":[163],"environment-level":[164],"performance.":[165],"Ablations":[166],"show":[167],"gains":[170],"arise":[171],"geometry":[174],"barrier":[178],"rather":[179],"than":[180],"flexible":[182],"boundaries":[184],"alone.":[185]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-05-13T00:00:00"}