{"id":"https://openalex.org/W7164042786","doi":"https://doi.org/10.48550/arxiv.2606.08340","title":"Benchmarking Open-Ended Multi-Agent Coordination in Language Agents","display_name":"Benchmarking Open-Ended Multi-Agent Coordination in Language Agents","publication_year":2026,"publication_date":"2026-06-06","ids":{"openalex":"https://openalex.org/W7164042786","doi":"https://doi.org/10.48550/arxiv.2606.08340"},"language":"en","primary_location":{"id":"pmh:oai:pure.ed.ac.uk:publications/dee46952-0cbf-46fc-b75d-8cb3badcf8af","is_oa":true,"landing_page_url":"https://www.research.ed.ac.uk/en/publications/dee46952-0cbf-46fc-b75d-8cb3badcf8af","pdf_url":null,"source":{"id":"https://openalex.org/S4306400321","display_name":"Edinburgh Research Explorer (University of Edinburgh)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98677209","host_organization_name":"University of Edinburgh","host_organization_lineage":["https://openalex.org/I98677209"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Tessera, K-A A, Szecsenyi, A, Barker, C, Rutherford, A, Paglieri, D, Scannell, A, Gouk, H, Storkey, A, Rockt\u00e4schel, T & Crowley, E J 2026 'Benchmarking open-ended multi-agent coordination in language agents' pp. 1-42. https://doi.org/10.48550/arXiv.2606.08340","raw_type":"info:eu-repo/semantics/preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.research.ed.ac.uk/en/publications/dee46952-0cbf-46fc-b75d-8cb3badcf8af","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048679971","display_name":"Kale-ab Tessera","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tessera, Kale-ab Abebe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125735834","display_name":"Andras Szecsenyi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Szecsenyi, Andras","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079353019","display_name":"Cameron Barker","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barker, Cameron","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138230292","display_name":"Alexander Rutherford","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rutherford, Alexander","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093796358","display_name":"Davide Paglieri","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paglieri, Davide","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075709964","display_name":"Aidan Scannell","orcid":"https://orcid.org/0000-0002-8393-7776"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Scannell, Aidan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136766875","display_name":"Henry Gouk","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gouk, Henry","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003510948","display_name":"Elliot J. Crowley","orcid":"https://orcid.org/0000-0001-5685-4724"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Crowley, Elliot J.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138277534","display_name":"Tim Rockt\u00e4schel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rockt\u00e4schel, Tim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136871004","display_name":"Amos Storkey","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Storkey, Amos","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.31279999017715454,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.31279999017715454,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.07530000060796738,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.07109999656677246,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.7912999987602234},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6003999710083008},{"id":"https://openalex.org/keywords/testbed","display_name":"Testbed","score":0.5501000285148621},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4196000099182129},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.41659998893737793},{"id":"https://openalex.org/keywords/pooling","display_name":"Pooling","score":0.39100000262260437},{"id":"https://openalex.org/keywords/homogeneous","display_name":"Homogeneous","score":0.3750999867916107},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.3424000144004822}],"concepts":[{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.7912999987602234},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7059999704360962},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6003999710083008},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.5501000285148621},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4196000099182129},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.41659998893737793},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.39100000262260437},{"id":"https://openalex.org/C66882249","wikidata":"https://www.wikidata.org/wiki/Q169336","display_name":"Homogeneous","level":2,"score":0.3750999867916107},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3634999990463257},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3463999927043915},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3424000144004822},{"id":"https://openalex.org/C171018156","wikidata":"https://www.wikidata.org/wiki/Q7370306","display_name":"Rotation formalisms in three dimensions","level":2,"score":0.3336000144481659},{"id":"https://openalex.org/C107257861","wikidata":"https://www.wikidata.org/wiki/Q656316","display_name":"Coordination game","level":2,"score":0.32749998569488525},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.3133000135421753},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.31200000643730164},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.30079999566078186},{"id":"https://openalex.org/C51929080","wikidata":"https://www.wikidata.org/wiki/Q2425187","display_name":"Codebase","level":3,"score":0.2978000044822693},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.29010000824928284},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.2870999872684479},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.27790001034736633},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.26460000872612},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.2637999951839447},{"id":"https://openalex.org/C13687954","wikidata":"https://www.wikidata.org/wiki/Q4826847","display_name":"Autonomous agent","level":2,"score":0.25110000371932983},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.25099998712539673}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:pure.ed.ac.uk:publications/dee46952-0cbf-46fc-b75d-8cb3badcf8af","is_oa":true,"landing_page_url":"https://www.research.ed.ac.uk/en/publications/dee46952-0cbf-46fc-b75d-8cb3badcf8af","pdf_url":null,"source":{"id":"https://openalex.org/S4306400321","display_name":"Edinburgh Research Explorer (University of Edinburgh)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98677209","host_organization_name":"University of Edinburgh","host_organization_lineage":["https://openalex.org/I98677209"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Tessera, K-A A, Szecsenyi, A, Barker, C, Rutherford, A, Paglieri, D, Scannell, A, Gouk, H, Storkey, A, Rockt\u00e4schel, T & Crowley, E J 2026 'Benchmarking open-ended multi-agent coordination in language agents' pp. 1-42. https://doi.org/10.48550/arXiv.2606.08340","raw_type":"info:eu-repo/semantics/preprint"},{"id":"doi:10.48550/arxiv.2606.08340","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08340","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:pure.ed.ac.uk:publications/dee46952-0cbf-46fc-b75d-8cb3badcf8af","is_oa":true,"landing_page_url":"https://www.research.ed.ac.uk/en/publications/dee46952-0cbf-46fc-b75d-8cb3badcf8af","pdf_url":null,"source":{"id":"https://openalex.org/S4306400321","display_name":"Edinburgh Research Explorer (University of Edinburgh)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98677209","host_organization_name":"University of Edinburgh","host_organization_lineage":["https://openalex.org/I98677209"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Tessera, K-A A, Szecsenyi, A, Barker, C, Rutherford, A, Paglieri, D, Scannell, A, Gouk, H, Storkey, A, Rockt\u00e4schel, T & Crowley, E J 2026 'Benchmarking open-ended multi-agent coordination in language agents' pp. 1-42. https://doi.org/10.48550/arXiv.2606.08340","raw_type":"info:eu-repo/semantics/preprint"},"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.5267648696899414}],"awards":[{"id":"https://openalex.org/G2360678933","display_name":"EPSRC Centre for Doctoral Training in Machine Learning Systems","funder_award_id":"EP/Y03516X/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G5730701950","display_name":null,"funder_award_id":"EP/W002876/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320320005","display_name":"Royal Academy of Engineering","ror":"https://ror.org/0526snb40"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"},{"id":"https://openalex.org/F4320335041","display_name":"School of Informatics, University of Edinburgh","ror":"https://ror.org/01nrxwf90"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"language":[1],"models":[2],"are":[3,110],"increasingly":[4],"deployed":[5],"as":[6,91,177],"autonomous":[7],"agents,":[8,184],"they":[9],"must":[10],"coordinate":[11],"with":[12,72,87],"others":[13],"over":[14],"long":[15],"horizons":[16],"in":[17],"open-ended":[18,47],"interactive":[19],"tasks.":[20],"Yet":[21],"existing":[22],"evaluations":[23],"rarely":[24],"test":[25],"these":[26],"demands":[27],"together,":[28],"instead":[29],"emphasising":[30],"single-agent":[31,187],"tasks,":[32,59],"short":[33],"interactions,":[34],"or":[35],"highly":[36],"structured":[37],"multi-agent":[38,48],"settings.":[39],"We":[40,78],"introduce":[41],"$alem$,":[42],"a":[43,68,178,196],"JAX-based":[44],"benchmark":[45],"for":[46,124,181,199],"coordination":[49,58,65,116,137,149,176],"built":[50],"on":[51],"Craftax-like":[52],"dynamics.":[53],"Alem":[54,189],"embeds":[55],"procedurally":[56],"generated":[57],"soft":[60],"specialisation,":[61],"communication,":[62],"and":[63,76,163,194,206],"controllable":[64],"difficulty":[66],"into":[67],"long-horizon":[69],"survival":[70],"world":[71],"exploration,":[73],"crafting,":[74],"trading,":[75],"combat.":[77],"evaluate":[79],"$13$":[80],"modern":[81],"LLMs":[82],"zero-shot":[83,118],"within":[84],"homogeneous":[85],"teams,":[86],"trained":[88,123],"MARL":[89,121],"agents":[90,96,122,201],"reference":[92],"points.":[93],"Current":[94],"LLM":[95,183],"remain":[97],"far":[98],"from":[99,186],"solving":[100],"alem,":[101],"averaging":[102],"only":[103],"~6%":[104],"normalised":[105],"return,":[106],"but":[107,134],"their":[108],"failures":[109],"not":[111,147],"uniform.":[112],"On":[113],"the":[114,156],"hardest":[115],"setting,":[117],"Gemini-3.1-Pro-High":[119],"approaches":[120],"one":[125],"billion":[126],"steps,":[127],"while":[128,161],"GPT-5.4-High":[129],"achieves":[130],"strong":[131],"base-task":[132],"reward":[133],"much":[135],"lower":[136],"reward.":[138],"This":[139],"contrast":[140],"shows":[141],"that":[142,153,202],"individual":[143],"task":[144],"competence":[145],"does":[146],"imply":[148],"competence.":[150],"Ablations":[151],"show":[152],"communication":[154],"is":[155,211],"largest":[157],"contributor":[158],"to":[159,168],"coordination,":[160],"memory":[162],"reasoning":[164],"help":[165],"when":[166],"used":[167],"maintain":[169],"multi-step":[170],"plans.":[171,209],"Overall,":[172],"our":[173],"results":[174],"identify":[175],"distinct":[179],"bottleneck":[180,192],"frontier":[182],"separate":[185],"capabilities.":[188],"makes":[190],"this":[191],"measurable":[193],"provides":[195],"controlled":[197],"testbed":[198],"developing":[200],"communicate,":[203],"allocate":[204],"roles,":[205],"execute":[207],"shared":[208],"Code":[210],"available":[212],"at":[213],"https://github.com/alem-world/alem-env.":[214]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-10T00:00:00"}