{"id":"https://openalex.org/W4382936050","doi":"https://doi.org/10.23919/acc55779.2023.10156072","title":"Scalable Multi-Agent Reinforcement Learning with General Utilities","display_name":"Scalable Multi-Agent Reinforcement Learning with General Utilities","publication_year":2023,"publication_date":"2023-05-31","ids":{"openalex":"https://openalex.org/W4382936050","doi":"https://doi.org/10.23919/acc55779.2023.10156072"},"language":"en","primary_location":{"id":"doi:10.23919/acc55779.2023.10156072","is_oa":false,"landing_page_url":"http://dx.doi.org/10.23919/acc55779.2023.10156072","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 American Control Conference (ACC)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065161741","display_name":"Donghao Ying","orcid":"https://orcid.org/0009-0001-7329-5917"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Donghao Ying","raw_affiliation_strings":["University of California at Berkeley,Department of Industrial Engineering and Operations Research","Department of Industrial Engineering and Operations Research, University of California at Berkeley"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California at Berkeley,Department of Industrial Engineering and Operations Research","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"Department of Industrial Engineering and Operations Research, University of California at Berkeley","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102879163","display_name":"Yuhao Ding","orcid":"https://orcid.org/0000-0002-1521-7566"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuhao Ding","raw_affiliation_strings":["University of California at Berkeley,Department of Industrial Engineering and Operations Research","Department of Industrial Engineering and Operations Research, University of California at Berkeley"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California at Berkeley,Department of Industrial Engineering and Operations Research","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"Department of Industrial Engineering and Operations Research, University of California at Berkeley","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025896653","display_name":"Alec Koppel","orcid":"https://orcid.org/0000-0003-2447-2873"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alec Koppel","raw_affiliation_strings":["J.P. Morgan AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"J.P. Morgan AI Research","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5042580848","display_name":"Javad Lavaei","orcid":null},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Javad Lavaei","raw_affiliation_strings":["University of California at Berkeley,Department of Industrial Engineering and Operations Research","Department of Industrial Engineering and Operations Research, University of California at Berkeley"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California at Berkeley,Department of Industrial Engineering and Operations Research","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"Department of Industrial Engineering and Operations Research, University of California at Berkeley","institution_ids":["https://openalex.org/I95457486"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3977","last_page":"3982"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10551","display_name":"Scheduling and Optimization Algorithms","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12288","display_name":"Optimization and Search Problems","score":0.9876000285148621,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.8711727261543274},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7798388600349426},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7147400379180908},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.41203010082244873},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3792499601840973},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.33466675877571106},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.32495754957199097},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.11802709102630615}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8711727261543274},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7798388600349426},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7147400379180908},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41203010082244873},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3792499601840973},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.33466675877571106},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.32495754957199097},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.11802709102630615}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.23919/acc55779.2023.10156072","is_oa":false,"landing_page_url":"http://dx.doi.org/10.23919/acc55779.2023.10156072","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 American Control Conference (ACC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.49000000953674316,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W855132903","https://openalex.org/W1486653676","https://openalex.org/W1507653385","https://openalex.org/W1596981350","https://openalex.org/W1988969816","https://openalex.org/W1997834106","https://openalex.org/W1999874108","https://openalex.org/W2100677568","https://openalex.org/W2113789941","https://openalex.org/W2120678009","https://openalex.org/W2155027007","https://openalex.org/W2505287782","https://openalex.org/W2584686170","https://openalex.org/W2886936744","https://openalex.org/W2953326529","https://openalex.org/W2963438456","https://openalex.org/W3038915804","https://openalex.org/W3132054471","https://openalex.org/W3202124232","https://openalex.org/W3207876914","https://openalex.org/W3209969885","https://openalex.org/W4283815983","https://openalex.org/W4287131900","https://openalex.org/W4382318857","https://openalex.org/W4382936050","https://openalex.org/W6683204974","https://openalex.org/W6748603076","https://openalex.org/W6757058172","https://openalex.org/W6758763022","https://openalex.org/W6765240361","https://openalex.org/W6780386840","https://openalex.org/W6791168167","https://openalex.org/W6796718340","https://openalex.org/W6801964087","https://openalex.org/W6802536806"],"related_works":["https://openalex.org/W4306904969","https://openalex.org/W2138720691","https://openalex.org/W4362501864","https://openalex.org/W1982914007","https://openalex.org/W2159583675","https://openalex.org/W1824242903","https://openalex.org/W1493858311","https://openalex.org/W2155470929","https://openalex.org/W2111125783","https://openalex.org/W2394465510"],"abstract_inverted_index":{"We":[0],"study":[1],"the":[2,16,32,35,41,48,52,58,123,128,132,144],"scalable":[3,64],"multi-agent":[4,135],"reinforcement":[5],"learning":[6],"(MARL)":[7],"with":[8,69,101,106,137],"general":[9,138],"utilities,":[10],"defined":[11],"as":[12],"nonlinear":[13],"functions":[14,39],"of":[15,34,44,57,77],"team\u2019s":[17,36],"long-term":[18],"state-action":[19],"occupancy":[20],"measure.":[21],"The":[22],"objective":[23],"is":[24,127],"to":[25,104,115],"find":[26],"a":[27,63],"localized":[28,73],"policy":[29,66,74,92,96],"that":[30,75,119,140],"maximizes":[31],"average":[33],"local":[37],"utility":[38],"without":[40],"full":[42,145],"observability":[43],"each":[45],"agent":[46],"in":[47,122,131],"team.":[49],"By":[50],"exploiting":[51],"spatial":[53],"correlation":[54],"decay":[55],"property":[56],"network":[59],"structure,":[60],"we":[61],"propose":[62],"distributed":[65],"gradient":[67,93],"algorithm":[68,99],"shadow":[70,81,86],"reward":[71,82],"and":[72,89,95],"consists":[76],"three":[78],"steps:":[79],"(1)":[80],"estimation,":[83,88],"(2)":[84],"truncated":[85,91],"Q-function":[87],"(3)":[90],"estimation":[94],"update.":[97],"Our":[98],"converges,":[100],"high":[102],"probability,":[103],"\u03f5-stationarity":[105],"$\\mathcal{O}\\left(":[107],"{{\\varepsilon":[108],"^{":[109],"-":[110],"2}}}":[111],"\\right)$":[112],"samples":[113],"up":[114],"some":[116],"approximation":[117],"error":[118],"decreases":[120],"exponentially":[121],"communication":[124],"radius.":[125],"This":[126],"first":[129],"result":[130],"literature":[133],"on":[134],"RL":[136],"utilities":[139],"does":[141],"not":[142],"require":[143],"observability.":[146]},"counts_by_year":[{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}