{"id":"https://openalex.org/W4386590850","doi":"https://doi.org/10.1109/tetci.2023.3309738","title":"Hierarchical Coordination Multi-Agent Reinforcement Learning With Spatio-Temporal Abstraction","display_name":"Hierarchical Coordination Multi-Agent Reinforcement Learning With Spatio-Temporal Abstraction","publication_year":2023,"publication_date":"2023-09-11","ids":{"openalex":"https://openalex.org/W4386590850","doi":"https://doi.org/10.1109/tetci.2023.3309738"},"language":"en","primary_location":{"id":"doi:10.1109/tetci.2023.3309738","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tetci.2023.3309738","pdf_url":null,"source":{"id":"https://openalex.org/S4210210251","display_name":"IEEE Transactions on Emerging Topics in Computational Intelligence","issn_l":"2471-285X","issn":["2471-285X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Emerging Topics in Computational Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053920875","display_name":"Tinghuai Ma","orcid":"https://orcid.org/0000-0003-2320-1692"},"institutions":[{"id":"https://openalex.org/I200845125","display_name":"Nanjing University of Information Science and Technology","ror":"https://ror.org/02y0rxk19","country_code":"CN","type":"education","lineage":["https://openalex.org/I200845125"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tinghuai Ma","raw_affiliation_strings":["School of Software, Nanjing University of Information Science & Technology, Nanjing, Jiangsu, China"],"raw_orcid":"https://orcid.org/0000-0003-2320-1692","affiliations":[{"raw_affiliation_string":"School of Software, Nanjing University of Information Science & Technology, Nanjing, Jiangsu, China","institution_ids":["https://openalex.org/I200845125"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056395476","display_name":"Kexing Peng","orcid":"https://orcid.org/0000-0002-2712-9088"},"institutions":[{"id":"https://openalex.org/I200845125","display_name":"Nanjing University of Information Science and Technology","ror":"https://ror.org/02y0rxk19","country_code":"CN","type":"education","lineage":["https://openalex.org/I200845125"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kexing Peng","raw_affiliation_strings":["School of Computer Science, Nanjing University of Information Science & Technology, Nanjing, Jiangsu, China"],"raw_orcid":"https://orcid.org/0000-0002-2712-9088","affiliations":[{"raw_affiliation_string":"School of Computer Science, Nanjing University of Information Science & Technology, Nanjing, Jiangsu, China","institution_ids":["https://openalex.org/I200845125"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041302956","display_name":"Huan Rong","orcid":"https://orcid.org/0000-0003-0542-1827"},"institutions":[{"id":"https://openalex.org/I200845125","display_name":"Nanjing University of Information Science and Technology","ror":"https://ror.org/02y0rxk19","country_code":"CN","type":"education","lineage":["https://openalex.org/I200845125"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huan Rong","raw_affiliation_strings":["School of Artificial Intelligence, Nanjing University of Information Science & Technology, Nanjing, Jiangsu, China"],"raw_orcid":"https://orcid.org/0000-0003-0542-1827","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Nanjing University of Information Science & Technology, Nanjing, Jiangsu, China","institution_ids":["https://openalex.org/I200845125"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002981402","display_name":"Yurong Qian","orcid":"https://orcid.org/0000-0001-6564-4745"},"institutions":[{"id":"https://openalex.org/I96908189","display_name":"Xinjiang University","ror":"https://ror.org/059gw8r13","country_code":"CN","type":"education","lineage":["https://openalex.org/I96908189"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yurong Qian","raw_affiliation_strings":["Xinjiang University, Urumqi, China"],"raw_orcid":"https://orcid.org/0000-0001-6564-4745","affiliations":[{"raw_affiliation_string":"Xinjiang University, Urumqi, China","institution_ids":["https://openalex.org/I96908189"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077894845","display_name":"Najla Al-Nabhan","orcid":"https://orcid.org/0000-0002-0805-1721"},"institutions":[{"id":"https://openalex.org/I28022161","display_name":"King Saud University","ror":"https://ror.org/02f81g417","country_code":"SA","type":"education","lineage":["https://openalex.org/I28022161"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Najla Al-Nabhan","raw_affiliation_strings":["Department of Computer Science, King Saud University, Riyadh, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0002-0805-1721","affiliations":[{"raw_affiliation_string":"Department of Computer Science, King Saud University, Riyadh, Saudi Arabia","institution_ids":["https://openalex.org/I28022161"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.4535,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.89108849,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"8","issue":"1","first_page":"533","last_page":"547"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10524","display_name":"Traffic control and management","score":0.9354000091552734,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10524","display_name":"Traffic control and management","score":0.9354000091552734,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7650548219680786},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7539047002792358},{"id":"https://openalex.org/keywords/observability","display_name":"Observability","score":0.7515993714332581},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.569085955619812},{"id":"https://openalex.org/keywords/abstraction","display_name":"Abstraction","score":0.5480993390083313},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.5465608835220337},{"id":"https://openalex.org/keywords/attention-network","display_name":"Attention network","score":0.4676741063594818},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4596514105796814},{"id":"https://openalex.org/keywords/information-sharing","display_name":"Information sharing","score":0.4195113778114319},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3754222095012665},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.36546820402145386},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.33984795212745667}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7650548219680786},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7539047002792358},{"id":"https://openalex.org/C36299963","wikidata":"https://www.wikidata.org/wiki/Q1369844","display_name":"Observability","level":2,"score":0.7515993714332581},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.569085955619812},{"id":"https://openalex.org/C124304363","wikidata":"https://www.wikidata.org/wiki/Q673661","display_name":"Abstraction","level":2,"score":0.5480993390083313},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5465608835220337},{"id":"https://openalex.org/C2993807640","wikidata":"https://www.wikidata.org/wiki/Q103709453","display_name":"Attention network","level":2,"score":0.4676741063594818},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4596514105796814},{"id":"https://openalex.org/C2776854237","wikidata":"https://www.wikidata.org/wiki/Q6031064","display_name":"Information sharing","level":2,"score":0.4195113778114319},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3754222095012665},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36546820402145386},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.33984795212745667},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tetci.2023.3309738","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tetci.2023.3309738","pdf_url":null,"source":{"id":"https://openalex.org/S4210210251","display_name":"IEEE Transactions on Emerging Topics in Computational Intelligence","issn_l":"2471-285X","issn":["2471-285X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Emerging Topics in Computational Intelligence","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Sustainable cities and communities","id":"https://metadata.un.org/sdg/11","score":0.75}],"awards":[{"id":"https://openalex.org/G1588614882","display_name":null,"funder_award_id":"62372243","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4906812084","display_name":null,"funder_award_id":"42175194","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":40,"referenced_works":["https://openalex.org/W2617547828","https://openalex.org/W2903709398","https://openalex.org/W2998367975","https://openalex.org/W3091889011","https://openalex.org/W4205771884","https://openalex.org/W4205898562","https://openalex.org/W4207021923","https://openalex.org/W4207072548","https://openalex.org/W4211202045","https://openalex.org/W4212875520","https://openalex.org/W4212996499","https://openalex.org/W4221046081","https://openalex.org/W4225090121","https://openalex.org/W4225832811","https://openalex.org/W4285137529","https://openalex.org/W4285295496","https://openalex.org/W4285585325","https://openalex.org/W4296437512","https://openalex.org/W4296913591","https://openalex.org/W4312576764","https://openalex.org/W4312644341","https://openalex.org/W6712181171","https://openalex.org/W6713411898","https://openalex.org/W6749304979","https://openalex.org/W6752089545","https://openalex.org/W6752380930","https://openalex.org/W6755069753","https://openalex.org/W6758714175","https://openalex.org/W6758846586","https://openalex.org/W6762491519","https://openalex.org/W6767151588","https://openalex.org/W6767317771","https://openalex.org/W6767327128","https://openalex.org/W6775529125","https://openalex.org/W6783196708","https://openalex.org/W6783812536","https://openalex.org/W6787618087","https://openalex.org/W6790614080","https://openalex.org/W6840107105","https://openalex.org/W7020879021"],"related_works":["https://openalex.org/W2046459260","https://openalex.org/W2967463586","https://openalex.org/W2765830098","https://openalex.org/W1971989957","https://openalex.org/W2517338020","https://openalex.org/W3157641275","https://openalex.org/W4312300846","https://openalex.org/W2615757685","https://openalex.org/W2098782634","https://openalex.org/W3000546688"],"abstract_inverted_index":{"Many":[0],"real-world":[1],"cooperative":[2],"problems":[3],"can":[4],"be":[5],"implemented":[6],"using":[7],"Multi-Agent":[8],"Reinforcement":[9],"Learning":[10],"(MARL)":[11],"techniques,":[12],"such":[13],"as":[14,220],"urban":[15],"traffic":[16],"control":[17],"or":[18],"multi-role":[19],"games.":[20],"However,":[21],"the":[22,39,44,51,92,103,114,125,130,133,140,144,149,178,194,199,229,241],"policy":[23,90,105,116,123,151],"learning":[24,172],"of":[25,30,41,46,132,201,210,231,243],"MARL":[26,59],"algorithms":[27,238],"contains":[28],"features":[29],"long-trajectory":[31,205],"training":[32,206],"and":[33,43,60,79,96,109,137,170,183,239],"partial":[34],"observability,":[35],"which":[36],"leads":[37],"to":[38,117,128,142,166,192,235],"sparsity":[40],"reward":[42,111,119],"lack":[45],"decision":[47],"information.":[48],"To":[49],"solve":[50],"above":[52,93,126,224],"issues,":[53],"this":[54],"article":[55],"studies":[56],"hierarchical":[57,73],"deep":[58],"proposes":[61],"a":[62,88,153,204],"novel":[63],"model":[64,156,162],"named":[65],"Hierarchical":[66],"Spatio-Temporal":[67,158,161],"Communication":[68],"Network":[69],"(HSTCN).":[70],"HSTCN":[71,202,232,244],"designs":[72,152],"policies":[74,136,175],"with":[75,139,157],"two-time":[76],"granularities:":[77],"high-level":[78,104,150],"low-level":[80,115],"policies.":[81],"All":[82],"agents":[83],"are":[84,218],"jointly":[85],"entered":[86],"into":[87],"joint":[89],"containing":[91],"two":[94],"policies,":[95],"each":[97],"has":[98],"its":[99],"execution":[100,135],"policy.":[101],"Specifically,":[102],"provides":[106],"intrinsic":[107],"goals":[108],"continuous":[110],"samples":[112],"for":[113],"alleviate":[118],"sparsity.":[120],"The":[121,160,223],"Low-level":[122],"absorbs":[124],"information":[127,169],"improve":[129],"efficiency":[131],"agents'":[134],"interact":[138],"environment":[141,207],"optimize":[143],"next":[145],"reward.":[146],"What's":[147],"more,":[148],"graph-like":[154],"structural":[155],"abstract.":[159],"expands":[163],"receptive":[164],"fields":[165],"receive":[167],"neighborhood":[168],"facilitates":[171],"more":[173],"robust":[174],"by":[176],"capturing":[177],"underlying":[179],"graph's":[180],"spatial":[181],"dependencies":[182],"temporal":[184],"dynamics.":[185],"Meanwhile,":[186],"an":[187],"evaluation":[188],"network":[189],"is":[190,233],"added":[191],"increase":[193],"robustness.":[195],"Empirically,":[196],"we":[197],"demonstrated":[198],"effectiveness":[200],"in":[203],"through":[208],"Simulation":[209],"Urban":[211],"MObility":[212],"(SUMO),":[213],"while":[214],"StarCraft":[215],"II":[216],"maps":[217],"tested":[219],"abstract":[221],"environment.":[222],"experimental":[225],"results":[226],"prove":[227],"that":[228],"performance":[230],"superior":[234],"other":[236],"advanced":[237],"verify":[240],"rationality":[242],"design.":[245]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}