{"id":"https://openalex.org/W7154705161","doi":"https://doi.org/10.48550/arxiv.2604.14564","title":"MARS$^2$: Scaling Multi-Agent Tree Search via Reinforcement Learning for Code Generation","display_name":"MARS$^2$: Scaling Multi-Agent Tree Search via Reinforcement Learning for Code Generation","publication_year":2026,"publication_date":"2026-04-16","ids":{"openalex":"https://openalex.org/W7154705161","doi":"https://doi.org/10.48550/arxiv.2604.14564"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.14564","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.14564","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.14564","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133837568","display_name":"Pengfei Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Li, Pengfei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133869725","display_name":"Shijie Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Shijie","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133868136","display_name":"Fangyuan Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Fangyuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133827985","display_name":"Yikun Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Yikun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133841371","display_name":"Kaifeng Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Kaifeng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133843846","display_name":"Kaiyan Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Kaiyan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038068859","display_name":"Dazhi Zhang","orcid":"https://orcid.org/0009-0005-0518-7702"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Dazhi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133828376","display_name":"Yuqiang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yuqiang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133863968","display_name":"Biqing Qi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi, Biqing","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133853238","display_name":"Bowen Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Bowen","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5133837568"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.2727999985218048,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.2727999985218048,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.1543000042438507,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11975","display_name":"Evolutionary Algorithms and Applications","score":0.0714000016450882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.8205999732017517},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5468000173568726},{"id":"https://openalex.org/keywords/tree","display_name":"Tree (set theory)","score":0.5088000297546387},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.3625999987125397},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.3488999903202057},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.32910001277923584},{"id":"https://openalex.org/keywords/search-algorithm","display_name":"Search algorithm","score":0.3287000060081482}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8205999732017517},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7699999809265137},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5468000173568726},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.5088000297546387},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.49140000343322754},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4675000011920929},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.3625999987125397},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3488999903202057},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.32910001277923584},{"id":"https://openalex.org/C125583679","wikidata":"https://www.wikidata.org/wiki/Q755673","display_name":"Search algorithm","level":2,"score":0.3287000060081482},{"id":"https://openalex.org/C2777866876","wikidata":"https://www.wikidata.org/wiki/Q5421358","display_name":"Exploratory search","level":2,"score":0.3262999951839447},{"id":"https://openalex.org/C207024777","wikidata":"https://www.wikidata.org/wiki/Q621673","display_name":"Search tree","level":3,"score":0.3124000132083893},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3109999895095825},{"id":"https://openalex.org/C2778915421","wikidata":"https://www.wikidata.org/wiki/Q3643177","display_name":"Performance improvement","level":2,"score":0.3010999858379364},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2915000021457672},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.2865999937057495},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.2800999879837036},{"id":"https://openalex.org/C19889080","wikidata":"https://www.wikidata.org/wiki/Q2835852","display_name":"Beam search","level":3,"score":0.2623000144958496}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.14564","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.14564","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.14564","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.14564","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"(RL)":[2],"paradigms":[3],"have":[4],"demonstrated":[5],"strong":[6],"performance":[7,28,151],"on":[8,128,142],"reasoning-intensive":[9],"tasks":[10],"such":[11],"as":[12,95],"code":[13,143,174],"generation.":[14],"However,":[15],"limited":[16],"trajectory":[17],"diversity":[18],"often":[19],"leads":[20],"to":[21,104],"diminishing":[22],"returns,":[23],"which":[24,39,79,132],"constrains":[25],"the":[26,43,92,160],"achievable":[27],"ceiling.":[29],"Search-enhanced":[30],"RL":[31,76],"alleviates":[32],"this":[33],"issue":[34],"by":[35,42],"introducing":[36],"structured":[37,65],"exploration,":[38],"remains":[40],"constrained":[41],"single-agent":[44],"policy":[45],"priors.":[46],"Meanwhile,":[47],"leveraging":[48],"multiple":[49,80],"interacting":[50],"policies":[51],"can":[52],"acquire":[53],"more":[54],"diverse":[55,153],"exploratory":[56],"signals,":[57],"but":[58],"existing":[59],"approaches":[60],"are":[61],"typically":[62],"decoupled":[63],"from":[64],"search.":[66],"We":[67],"propose":[68],"\\textbf{MARS$^2$}":[69],"(Multi-Agent":[70],"Reinforced":[71],"Tree-Search":[72],"Scaling),":[73],"a":[74,85,96,112,122],"unified":[75],"framework":[77],"in":[78],"independently-optimized":[81],"agents":[82,103],"collaborate":[83],"within":[84,111],"shared":[86,113],"tree-structured":[87],"search":[88,93,114,139,168],"environment.":[89],"MARS$^2$":[90,148],"models":[91],"tree":[94,167],"learnable":[97],"multi-agent":[98,164],"interaction":[99],"environment,":[100],"enabling":[101],"heterogeneous":[102],"collaboratively":[105],"generate":[106],"and":[107,156],"refine":[108],"candidate":[109],"solutions":[110],"topology.":[115],"To":[116],"support":[117],"effective":[118,134],"learning,":[119],"we":[120],"introduce":[121],"path-level":[123],"group":[124],"advantage":[125],"formulation":[126],"based":[127],"tree-consistent":[129],"reward":[130],"shaping,":[131],"facilitates":[133],"credit":[135],"assignment":[136],"across":[137,152],"complex":[138],"trajectories.":[140],"Experiments":[141],"generation":[144],"benchmarks":[145],"show":[146],"that":[147],"consistently":[149],"improves":[150],"model":[154],"combinations":[155],"training":[157],"settings,":[158],"demonstrating":[159],"effectiveness":[161],"of":[162],"coupling":[163],"collaboration":[165],"with":[166],"for":[169],"enhancing":[170],"reinforcement":[171],"learning.":[172],"Our":[173],"is":[175],"publicly":[176],"available":[177],"at":[178],"https://github.com/TsinghuaC3I/MARTI.":[179]},"counts_by_year":[],"updated_date":"2026-04-18T06:05:20.339008","created_date":"2026-04-18T00:00:00"}