{"id":"https://openalex.org/W4395030794","doi":"https://doi.org/10.1007/s40747-024-01415-1","title":"GHQ: grouped hybrid Q-learning for cooperative heterogeneous multi-agent reinforcement learning","display_name":"GHQ: grouped hybrid Q-learning for cooperative heterogeneous multi-agent reinforcement learning","publication_year":2024,"publication_date":"2024-04-23","ids":{"openalex":"https://openalex.org/W4395030794","doi":"https://doi.org/10.1007/s40747-024-01415-1"},"language":"en","primary_location":{"id":"doi:10.1007/s40747-024-01415-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40747-024-01415-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40747-024-01415-1.pdf","source":{"id":"https://openalex.org/S3035462843","display_name":"Complex & Intelligent Systems","issn_l":"2198-6053","issn":["2198-6053","2199-4536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Complex & Intelligent Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://link.springer.com/content/pdf/10.1007/s40747-024-01415-1.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Xiaoyang Yu","orcid":"https://orcid.org/0000-0001-6533-5176"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xiaoyang Yu","raw_affiliation_strings":["Beijing Key Laboratory of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing, China","School of Computer and Information Technology, Beijing Jiaotong University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6533-5176","affiliations":[{"raw_affiliation_string":"Beijing Key Laboratory of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]},{"raw_affiliation_string":"School of Computer and Information Technology, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102790320","display_name":"Youfang Lin","orcid":"https://orcid.org/0000-0002-1611-4323"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Youfang Lin","raw_affiliation_strings":["Beijing Key Laboratory of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing, China","School of Computer and Information Technology, Beijing Jiaotong University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing Key Laboratory of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]},{"raw_affiliation_string":"School of Computer and Information Technology, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064807299","display_name":"Xiangsen Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiangsen Wang","raw_affiliation_strings":["Beijing Key Laboratory of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing, China","School of Computer and Information Technology, Beijing Jiaotong University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing Key Laboratory of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]},{"raw_affiliation_string":"School of Computer and Information Technology, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101935099","display_name":"Sheng Han","orcid":"https://orcid.org/0000-0002-1212-2676"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sheng Han","raw_affiliation_strings":["Beijing Key Laboratory of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing, China","School of Computer and Information Technology, Beijing Jiaotong University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing Key Laboratory of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]},{"raw_affiliation_string":"School of Computer and Information Technology, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5026666435","display_name":"Kai Lv","orcid":"https://orcid.org/0000-0001-6533-5176"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kai Lv","raw_affiliation_strings":["Beijing Key Laboratory of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing, China","School of Computer and Information Technology, Beijing Jiaotong University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing Key Laboratory of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]},{"raw_affiliation_string":"School of Computer and Information Technology, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I21193070"],"apc_list":{"value":1320,"currency":"GBP","value_usd":1619},"apc_paid":{"value":1320,"currency":"GBP","value_usd":1619},"fwci":1.9868,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.87767006,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"10","issue":"4","first_page":"5261","last_page":"5280"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.9908999800682068,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7077987194061279},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5773045420646667},{"id":"https://openalex.org/keywords/marl","display_name":"Marl","score":0.5583434700965881},{"id":"https://openalex.org/keywords/mainstream","display_name":"Mainstream","score":0.5226097702980042},{"id":"https://openalex.org/keywords/homogeneous","display_name":"Homogeneous","score":0.500089168548584},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45650041103363037},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.3556942641735077},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3404543697834015},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2942502498626709},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.08323290944099426}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7077987194061279},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5773045420646667},{"id":"https://openalex.org/C92927620","wikidata":"https://www.wikidata.org/wiki/Q184053","display_name":"Marl","level":3,"score":0.5583434700965881},{"id":"https://openalex.org/C2777617010","wikidata":"https://www.wikidata.org/wiki/Q18957","display_name":"Mainstream","level":2,"score":0.5226097702980042},{"id":"https://openalex.org/C66882249","wikidata":"https://www.wikidata.org/wiki/Q169336","display_name":"Homogeneous","level":2,"score":0.500089168548584},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45650041103363037},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3556942641735077},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3404543697834015},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2942502498626709},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.08323290944099426},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C109007969","wikidata":"https://www.wikidata.org/wiki/Q749565","display_name":"Structural basin","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/s40747-024-01415-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40747-024-01415-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40747-024-01415-1.pdf","source":{"id":"https://openalex.org/S3035462843","display_name":"Complex & Intelligent Systems","issn_l":"2198-6053","issn":["2198-6053","2199-4536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Complex & Intelligent Systems","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:a74f8afa6d13485eb08f79e56b0adb5b","is_oa":true,"landing_page_url":"https://doaj.org/article/a74f8afa6d13485eb08f79e56b0adb5b","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Complex & Intelligent Systems, Vol 10, Iss 4, Pp 5261-5280 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1007/s40747-024-01415-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40747-024-01415-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40747-024-01415-1.pdf","source":{"id":"https://openalex.org/S3035462843","display_name":"Complex & Intelligent Systems","issn_l":"2198-6053","issn":["2198-6053","2199-4536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Complex & Intelligent Systems","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.5400000214576721}],"awards":[],"funders":[{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4395030794.pdf"},"referenced_works_count":58,"referenced_works":["https://openalex.org/W1641379095","https://openalex.org/W1981276685","https://openalex.org/W2000042664","https://openalex.org/W2096166399","https://openalex.org/W2157331557","https://openalex.org/W2292533394","https://openalex.org/W2531809222","https://openalex.org/W2617547828","https://openalex.org/W2747213132","https://openalex.org/W2768629321","https://openalex.org/W2903731519","https://openalex.org/W2907382309","https://openalex.org/W2909611215","https://openalex.org/W2968526727","https://openalex.org/W2981038142","https://openalex.org/W2990901095","https://openalex.org/W3000445347","https://openalex.org/W3012336324","https://openalex.org/W3042612721","https://openalex.org/W3042693249","https://openalex.org/W3086427656","https://openalex.org/W3093019345","https://openalex.org/W3094493698","https://openalex.org/W3115989973","https://openalex.org/W3121258823","https://openalex.org/W3127342281","https://openalex.org/W3133774413","https://openalex.org/W3135123930","https://openalex.org/W3156295478","https://openalex.org/W3178092445","https://openalex.org/W3196654803","https://openalex.org/W3202484768","https://openalex.org/W3209420330","https://openalex.org/W4200351942","https://openalex.org/W4206500519","https://openalex.org/W4282021251","https://openalex.org/W4282914661","https://openalex.org/W4283714159","https://openalex.org/W4283789768","https://openalex.org/W4285606083","https://openalex.org/W4285613784","https://openalex.org/W4290652759","https://openalex.org/W4297098388","https://openalex.org/W4308361232","https://openalex.org/W4308889875","https://openalex.org/W4312720137","https://openalex.org/W4313537271","https://openalex.org/W4319663730","https://openalex.org/W4321242409","https://openalex.org/W4386432258","https://openalex.org/W4393148408","https://openalex.org/W6600135713","https://openalex.org/W6600137863","https://openalex.org/W6600474606","https://openalex.org/W6601936271","https://openalex.org/W6602203279","https://openalex.org/W6602646459","https://openalex.org/W6602728322"],"related_works":["https://openalex.org/W2126019709","https://openalex.org/W1702901972","https://openalex.org/W4249798507","https://openalex.org/W2069775250","https://openalex.org/W2093541819","https://openalex.org/W2032294417","https://openalex.org/W2152754392","https://openalex.org/W1989172970","https://openalex.org/W2196316523","https://openalex.org/W1899363654"],"abstract_inverted_index":{"Abstract":[0],"Previous":[1],"deep":[2],"multi-agent":[3],"reinforcement":[4],"learning":[5],"(MARL)":[6],"algorithms":[7,59,131],"have":[8],"achieved":[9],"impressive":[10],"results,":[11],"typically":[12],"in":[13,42,102,121,136,193],"symmetric":[14],"and":[15,24,77,111,149,164,203],"homogeneous":[16],"scenarios.":[17],"However,":[18,71],"asymmetric":[19,43,118],"heterogeneous":[20,39,44,63,119],"scenarios":[21],"are":[22,123],"prevalent":[23],"usually":[25],"harder":[26],"to":[27,60,133,214],"solve.":[28],"In":[29],"this":[30],"paper,":[31],"the":[32,37,47,62,81,89,99,113,137,141,144,178,201,204,208],"main":[33],"discussion":[34],"is":[35,95,195],"about":[36],"cooperative":[38],"MARL":[40,64,152],"problem":[41,65,94,101],"maps":[45,120,206],"of":[46,80,88,183,211],"Starcraft":[48],"Multi-Agent":[49],"Challenges":[50],"(SMAC)":[51],"environment.":[52],"Recent":[53],"mainstream":[54],"approaches":[55,73],"use":[56],"policy-based":[57],"actor-critic":[58],"solve":[61],"with":[66],"various":[67],"individual":[68,166],"agent":[69],"policies.":[70],"these":[72],"lack":[74],"formal":[75,86],"definition":[76,87],"further":[78],"analysis":[79],"heterogeneity":[82],"problem.":[83],"Therefore,":[84],"a":[85,150],"Local":[90],"Transition":[91],"Heterogeneity":[92],"(LTH)":[93],"first":[96],"given.":[97],"Then,":[98,140],"LTH":[100,114],"SMAC":[103,122],"environment":[104],"can":[105],"be":[106],"studied.":[107],"To":[108,171],"comprehensively":[109],"reveal":[110],"study":[112],"problem,":[115],"some":[116],"new":[117,138,205],"designed.":[124],"It":[125],"has":[126],"been":[127],"observed":[128],"that":[129],"baseline":[130],"fail":[132],"perform":[134],"well":[135],"maps.":[139],"authors":[142],"propose":[143],"Grouped":[145,154],"Individual-Global-Max":[146],"(GIGM)":[147],"consistency":[148],"novel":[151,187],"algorithm,":[153],"Hybrid":[155],"Q-Learning":[156],"(GHQ).":[157],"GHQ":[158,176,194,212],"separates":[159],"agents":[160],"into":[161],"several":[162],"groups":[163],"keeps":[165],"parameters":[167],"for":[168,190],"each":[169],"group.":[170],"enhance":[172],"cooperation":[173],"between":[174,181],"groups,":[175],"maximizes":[177],"mutual":[179],"information":[180],"trajectories":[182],"different":[184],"groups.":[185],"A":[186],"hybrid":[188],"structure":[189],"value":[191],"factorization":[192],"also":[196],"proposed.":[197],"Finally,":[198],"experiments":[199],"on":[200],"original":[202],"show":[207],"fabulous":[209],"performance":[210],"compared":[213],"other":[215],"state-of-the-art":[216],"algorithms.":[217]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4}],"updated_date":"2026-02-03T23:48:18.283914","created_date":"2024-04-24T00:00:00"}