{"id":"https://openalex.org/W7111231663","doi":"https://doi.org/10.5555/3666122.3667756","title":"SMACv2:An Improved Benchmark for Cooperative Multi-Agent Reinforcement Learning","display_name":"SMACv2:An Improved Benchmark for Cooperative Multi-Agent Reinforcement Learning","publication_year":2024,"publication_date":"2024-07-01","ids":{"openalex":"https://openalex.org/W7111231663","doi":"https://doi.org/10.5555/3666122.3667756"},"language":"en","primary_location":{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/5c13c294-826a-4b8b-899f-424b460af5c0","doi":"https://doi.org/10.5555/3666122.3667756","is_oa":true,"landing_page_url":"https://research.manchester.ac.uk/en/publications/5c13c294-826a-4b8b-899f-424b460af5c0","pdf_url":null,"source":{"id":"https://openalex.org/S4306400662","display_name":"Research Explorer (The University of Manchester)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I28407311","host_organization_name":"University of Manchester","host_organization_lineage":["https://openalex.org/I28407311"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},"volume":null,"type":"conference-paper","indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://research.manchester.ac.uk/en/publications/5c13c294-826a-4b8b-899f-424b460af5c0","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5153876233","display_name":"Benjamin Ellis","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2801038905","display_name":"Mount Vernon Hospital","ror":"https://ror.org/04am5a125","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I2799786270","https://openalex.org/I2801038905"]},{"id":"https://openalex.org/I4210161180","display_name":"Mount Vernon Hospital","ror":"https://ror.org/00w9htx78","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I4210161180"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Ellis, Benjamin","raw_affiliation_strings":["Mount Vernon Hospital"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mount Vernon Hospital","institution_ids":["https://openalex.org/I2801038905","https://openalex.org/I4210161180"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5146747833","display_name":"Jonathan Cook","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2801038905","display_name":"Mount Vernon Hospital","ror":"https://ror.org/04am5a125","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I2799786270","https://openalex.org/I2801038905"]},{"id":"https://openalex.org/I4210161180","display_name":"Mount Vernon Hospital","ror":"https://ror.org/00w9htx78","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I4210161180"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Cook, Jonathan","raw_affiliation_strings":["Mount Vernon Hospital"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mount Vernon Hospital","institution_ids":["https://openalex.org/I2801038905","https://openalex.org/I4210161180"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5096130904","display_name":"Skander Moalla","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2801038905","display_name":"Mount Vernon Hospital","ror":"https://ror.org/04am5a125","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I2799786270","https://openalex.org/I2801038905"]},{"id":"https://openalex.org/I4210161180","display_name":"Mount Vernon Hospital","ror":"https://ror.org/00w9htx78","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I4210161180"]},{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH","GB"],"is_corresponding":false,"raw_author_name":"Moalla, Skander","raw_affiliation_strings":["Mount Vernon Hospital","Swiss Federal Institute of Technology, Lausanne"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mount Vernon Hospital","institution_ids":["https://openalex.org/I2801038905","https://openalex.org/I4210161180"]},{"raw_affiliation_string":"Swiss Federal Institute of Technology, Lausanne","institution_ids":["https://openalex.org/I5124864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016880022","display_name":"Mikayel Samvelyan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210147770","display_name":"Mortimer Market Centre","ror":"https://ror.org/056hsfz11","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I21862455","https://openalex.org/I4210147770"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Samvelyan, Mikayel","raw_affiliation_strings":["Meta Platforms Technologies LLC","Mortimer Market Centre"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta Platforms Technologies LLC","institution_ids":[]},{"raw_affiliation_string":"Mortimer Market Centre","institution_ids":["https://openalex.org/I4210147770"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101591811","display_name":"Mingfei Sun","orcid":"https://orcid.org/0000-0002-6652-8751","observed_orcids":["https://orcid.org/0000-0002-6652-8751"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Mingfei; id_orcid 0000-0002-5925-5425","raw_affiliation_strings":["Machine Learning and Robotics"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Machine Learning and Robotics","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103030202","display_name":"Anuj Mahajan","orcid":"https://orcid.org/0000-0003-2229-5287","observed_orcids":["https://orcid.org/0000-0003-2229-5287"]},"institutions":[{"id":"https://openalex.org/I2801038905","display_name":"Mount Vernon Hospital","ror":"https://ror.org/04am5a125","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I2799786270","https://openalex.org/I2801038905"]},{"id":"https://openalex.org/I4210161180","display_name":"Mount Vernon Hospital","ror":"https://ror.org/00w9htx78","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I4210161180"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mahajan, Anuj","raw_affiliation_strings":["Mount Vernon Hospital"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mount Vernon Hospital","institution_ids":["https://openalex.org/I2801038905","https://openalex.org/I4210161180"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130091962","display_name":"Jakob N. Foerster","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2801038905","display_name":"Mount Vernon Hospital","ror":"https://ror.org/04am5a125","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I2799786270","https://openalex.org/I2801038905"]},{"id":"https://openalex.org/I4210161180","display_name":"Mount Vernon Hospital","ror":"https://ror.org/00w9htx78","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I4210161180"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Foerster, Jakob N.","raw_affiliation_strings":["Mount Vernon Hospital"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mount Vernon Hospital","institution_ids":["https://openalex.org/I2801038905","https://openalex.org/I4210161180"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056879203","display_name":"Shimon Whiteson","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2801038905","display_name":"Mount Vernon Hospital","ror":"https://ror.org/04am5a125","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I2799786270","https://openalex.org/I2801038905"]},{"id":"https://openalex.org/I4210161180","display_name":"Mount Vernon Hospital","ror":"https://ror.org/00w9htx78","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I4210161180"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Whiteson, Shimon","raw_affiliation_strings":["Mount Vernon Hospital"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mount Vernon Hospital","institution_ids":["https://openalex.org/I2801038905","https://openalex.org/I4210161180"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5153876233"],"corresponding_institution_ids":["https://openalex.org/I2801038905","https://openalex.org/I4210161180"],"apc_list":null,"apc_paid":null,"fwci":4.9932,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.96558658,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.002899999963119626,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/starcraft-multi-agent-challenge","display_name":"StarCraft Multi-Agent Challenge","score":0.9120000004768372},{"id":"https://openalex.org/keywords/cooperative-multi-agent-reinforcement-learning","display_name":"cooperative multi-agent reinforcement learning","score":0.9639999866485596},{"id":"https://openalex.org/keywords/benchmark-design","display_name":"benchmark design","score":0.9330000281333923},{"id":"https://openalex.org/keywords/partial-observability","display_name":"partial observability","score":0.8600000143051147},{"id":"https://openalex.org/keywords/real-time-strategy-games","display_name":"real-time strategy games","score":0.6079999804496765}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.895799994468689},{"id":"https://openalex.org/C36299963","wikidata":"https://www.wikidata.org/wiki/Q1369844","display_name":"Observability","level":2,"score":0.7828999757766724},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7577999830245972},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6801999807357788},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.6535999774932861},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5285999774932861},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5170000195503235},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4975999891757965},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.4884999990463257},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.2754000127315521},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.2531999945640564},{"id":"https://openalex.org/C88626702","wikidata":"https://www.wikidata.org/wiki/Q1128903","display_name":"Continuation","level":2,"score":0.25279998779296875}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/5c13c294-826a-4b8b-899f-424b460af5c0","doi":"https://doi.org/10.5555/3666122.3667756","is_oa":true,"landing_page_url":"https://research.manchester.ac.uk/en/publications/5c13c294-826a-4b8b-899f-424b460af5c0","pdf_url":null,"source":{"id":"https://openalex.org/S4306400662","display_name":"Research Explorer (The University of Manchester)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I28407311","host_organization_name":"University of Manchester","host_organization_lineage":["https://openalex.org/I28407311"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/5c13c294-826a-4b8b-899f-424b460af5c0","doi":"https://doi.org/10.5555/3666122.3667756","is_oa":true,"landing_page_url":"https://research.manchester.ac.uk/en/publications/5c13c294-826a-4b8b-899f-424b460af5c0","pdf_url":null,"source":{"id":"https://openalex.org/S4306400662","display_name":"Research Explorer (The University of Manchester)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I28407311","host_organization_name":"University of Manchester","host_organization_lineage":["https://openalex.org/I28407311"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2913662073","https://openalex.org/W4405768320","https://openalex.org/W4376123175","https://openalex.org/W7124308745","https://openalex.org/W4415299510","https://openalex.org/W7124263640","https://openalex.org/W4284692283","https://openalex.org/W3176840732","https://openalex.org/W4417530308","https://openalex.org/W7081972261"],"abstract_inverted_index":{"The":[0],"availability":[1],"of":[2,14,41,104,155,186,195],"challenging":[3],"benchmarks":[4],"has":[5,27],"played":[6],"a":[7,29,101],"key":[8],"role":[9],"in":[10,173],"the":[11,22,62,83,105,122,130,150,153,174,183,192],"recent":[12],"progress":[13],"machine":[15],"learning.":[16],"In":[17,51,72],"cooperative":[18],"multi-agent":[19],"reinforcement":[20],"learning,":[21],"StarCraft":[23],"Multi-Agent":[24],"Challenge":[25],"(SMAC)":[26],"become":[28],"popular":[30],"testbed":[31],"for":[32,90],"centralised":[33],"training":[34],"with":[35],"decentralised":[36],"execution.":[37],"However,":[38],"after":[39],"years":[40],"sustained":[42],"improvement":[43],"on":[44,82,162],"SMAC,":[45],"algorithms":[46,161],"now":[47],"achieve":[48,86],"near-perfect":[49],"performance.":[50],"this":[52,96],"work,":[53],"we":[54,74,98],"conduct":[55],"new":[56,102],"analysis":[57,178],"demonstrating":[58],"that":[59,76,146,166,180],"SMAC":[60,92,187],"lacks":[61],"stochasticity":[63],"and":[64,112,164,188],"partial":[65,132,142],"observability":[66,133],"to":[67,115,117,139],"require":[68,113],"complex":[69],"closed-loop":[70,156],"policies.":[71,157],"particular,":[73],"show":[75,145,165],"an":[77],"open-loop":[78],"policy":[79],"conditioned":[80],"only":[81],"timestep":[84],"can":[85,189],"non-trivial":[87],"win":[88],"rates":[89],"many":[91],"scenarios.":[93],"To":[94],"address":[95],"limitation,":[97],"introduce":[99,129],"SMACv2,":[100],"version":[103],"benchmark":[106,151,191],"where":[107],"scenarios":[108],"are":[109],"procedurally":[110],"generated":[111],"agents":[114],"generalise":[116],"previously":[118],"unseen":[119],"settings":[120],"(from":[121],"same":[123],"distribution)":[124],"during":[125],"evaluation.":[126],"We":[127,144,158],"also":[128],"extended":[131],"challenge":[134],"(EPO),":[135],"which":[136],"augments":[137],"SMACv2":[138,163,181],"ensure":[140,149],"meaningful":[141],"observability.":[143],"these":[147],"changes":[148],"requires":[152],"use":[154],"evaluate":[159],"state-of-the-art":[160],"it":[167],"presents":[168],"significant":[169],"challenges":[170],"not":[171],"present":[172],"original":[175],"benchmark.":[176],"Our":[177],"illustrates":[179],"addresses":[182],"discovered":[184],"deficiencies":[185],"help":[190],"next":[193],"generation":[194],"MARL":[196],"methods.":[197]},"counts_by_year":[{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":1}],"updated_date":"2026-10-30T12:39:25.321706","created_date":"2025-12-10T00:00:00"}