{"id":"https://openalex.org/W4322704167","doi":"https://doi.org/10.1109/tsg.2023.3250321","title":"Hierarchical Hybrid Multi-Agent Deep Reinforcement Learning for Peer-to-Peer Energy Trading Among Multiple Heterogeneous Microgrids","display_name":"Hierarchical Hybrid Multi-Agent Deep Reinforcement Learning for Peer-to-Peer Energy Trading Among Multiple Heterogeneous Microgrids","publication_year":2023,"publication_date":"2023-02-28","ids":{"openalex":"https://openalex.org/W4322704167","doi":"https://doi.org/10.1109/tsg.2023.3250321"},"language":"en","primary_location":{"id":"doi:10.1109/tsg.2023.3250321","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tsg.2023.3250321","pdf_url":null,"source":{"id":"https://openalex.org/S59604973","display_name":"IEEE Transactions on Smart Grid","issn_l":"1949-3053","issn":["1949-3053","1949-3061"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Smart Grid","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088887359","display_name":"Yuxin Wu","orcid":"https://orcid.org/0000-0002-6910-7397"},"institutions":[{"id":"https://openalex.org/I159948400","display_name":"Jinan University","ror":"https://ror.org/02xe5ns62","country_code":"CN","type":"education","lineage":["https://openalex.org/I159948400"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yuxin Wu","raw_affiliation_strings":["College of Information Science and Technology, Jinan University, Guangzhou, China","Energy Electricity Research Center, Jinan University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0002-6910-7397","affiliations":[{"raw_affiliation_string":"College of Information Science and Technology, Jinan University, Guangzhou, China","institution_ids":["https://openalex.org/I159948400"]},{"raw_affiliation_string":"Energy Electricity Research Center, Jinan University, Zhuhai, China","institution_ids":["https://openalex.org/I159948400"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031912756","display_name":"Tianyang Zhao","orcid":"https://orcid.org/0000-0003-0184-6553"},"institutions":[{"id":"https://openalex.org/I159948400","display_name":"Jinan University","ror":"https://ror.org/02xe5ns62","country_code":"CN","type":"education","lineage":["https://openalex.org/I159948400"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianyang Zhao","raw_affiliation_strings":["Energy Electricity Research Center, Jinan University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0003-0184-6553","affiliations":[{"raw_affiliation_string":"Energy Electricity Research Center, Jinan University, Zhuhai, China","institution_ids":["https://openalex.org/I159948400"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065261587","display_name":"Haoyuan Yan","orcid":"https://orcid.org/0000-0002-8739-262X"},"institutions":[{"id":"https://openalex.org/I159948400","display_name":"Jinan University","ror":"https://ror.org/02xe5ns62","country_code":"CN","type":"education","lineage":["https://openalex.org/I159948400"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haoyuan Yan","raw_affiliation_strings":["Energy Electricity Research Center, Jinan University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0002-8739-262X","affiliations":[{"raw_affiliation_string":"Energy Electricity Research Center, Jinan University, Zhuhai, China","institution_ids":["https://openalex.org/I159948400"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100785551","display_name":"Min Liu","orcid":"https://orcid.org/0000-0002-1086-9637"},"institutions":[{"id":"https://openalex.org/I159948400","display_name":"Jinan University","ror":"https://ror.org/02xe5ns62","country_code":"CN","type":"education","lineage":["https://openalex.org/I159948400"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Liu","raw_affiliation_strings":["Energy Electricity Research Center, Jinan University, Zhuhai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Energy Electricity Research Center, Jinan University, Zhuhai, China","institution_ids":["https://openalex.org/I159948400"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100377349","display_name":"Nian Liu","orcid":"https://orcid.org/0000-0001-5971-7995"},"institutions":[{"id":"https://openalex.org/I153473198","display_name":"North China Electric Power University","ror":"https://ror.org/04qr5t414","country_code":"CN","type":"education","lineage":["https://openalex.org/I153473198"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Nian Liu","raw_affiliation_strings":["State Key Laboratory of Alternate Electrical Power System With Renewable Energy Sources, North China Electric Power University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5971-7995","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Alternate Electrical Power System With Renewable Energy Sources, North China Electric Power University, Beijing, China","institution_ids":["https://openalex.org/I153473198"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5088887359"],"corresponding_institution_ids":["https://openalex.org/I159948400"],"apc_list":null,"apc_paid":null,"fwci":9.5713,"has_fulltext":false,"cited_by_count":75,"citation_normalized_percentile":{"value":0.98667545,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"14","issue":"6","first_page":"4649","last_page":"4665"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10603","display_name":"Smart Grid Energy Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10603","display_name":"Smart Grid Energy Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10223","display_name":"Microgrid Control and Optimization","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10768","display_name":"Electric Vehicles and Infrastructure","score":0.9830999970436096,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.789993166923523},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7044060230255127},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6721911430358887},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6383655667304993},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.6060848236083984},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.574099600315094},{"id":"https://openalex.org/keywords/job-shop-scheduling","display_name":"Job shop scheduling","score":0.41576170921325684},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.34621763229370117},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.29479852318763733},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.1251349151134491}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.789993166923523},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7044060230255127},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6721911430358887},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6383655667304993},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.6060848236083984},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.574099600315094},{"id":"https://openalex.org/C55416958","wikidata":"https://www.wikidata.org/wiki/Q6206757","display_name":"Job shop scheduling","level":3,"score":0.41576170921325684},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.34621763229370117},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29479852318763733},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.1251349151134491},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tsg.2023.3250321","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tsg.2023.3250321","pdf_url":null,"source":{"id":"https://openalex.org/S59604973","display_name":"IEEE Transactions on Smart Grid","issn_l":"1949-3053","issn":["1949-3053","1949-3061"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Smart Grid","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Affordable and clean energy","id":"https://metadata.un.org/sdg/7","score":0.5699999928474426}],"awards":[{"id":"https://openalex.org/G4565139328","display_name":null,"funder_award_id":"LAPS21002","funder_id":"https://openalex.org/F4320327677","funder_display_name":"State Key Laboratory Of Alternate Electrical Power System With Renewable Energy Sources"},{"id":"https://openalex.org/G7886376936","display_name":null,"funder_award_id":"52061635102","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320327677","display_name":"State Key Laboratory Of Alternate Electrical Power System With Renewable Energy Sources","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":67,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W1533576432","https://openalex.org/W1542941925","https://openalex.org/W2026080429","https://openalex.org/W2138289719","https://openalex.org/W2145339207","https://openalex.org/W2164278908","https://openalex.org/W2475218214","https://openalex.org/W2526471659","https://openalex.org/W2569201729","https://openalex.org/W2613514429","https://openalex.org/W2629433172","https://openalex.org/W2777216400","https://openalex.org/W2786291477","https://openalex.org/W2791029640","https://openalex.org/W2801383047","https://openalex.org/W2801389981","https://openalex.org/W2884722687","https://openalex.org/W2890184754","https://openalex.org/W2896967770","https://openalex.org/W2897091017","https://openalex.org/W2952326029","https://openalex.org/W2968869059","https://openalex.org/W2970077368","https://openalex.org/W2974854012","https://openalex.org/W2980533241","https://openalex.org/W2989302924","https://openalex.org/W2997360310","https://openalex.org/W3016141087","https://openalex.org/W3031637884","https://openalex.org/W3034596777","https://openalex.org/W3038624044","https://openalex.org/W3041621816","https://openalex.org/W3049473885","https://openalex.org/W3080812778","https://openalex.org/W3094401630","https://openalex.org/W3097422301","https://openalex.org/W3098532584","https://openalex.org/W3104894936","https://openalex.org/W3107222651","https://openalex.org/W3118326706","https://openalex.org/W3126304874","https://openalex.org/W3129869491","https://openalex.org/W3133923776","https://openalex.org/W3134572472","https://openalex.org/W3135349110","https://openalex.org/W3139861876","https://openalex.org/W3153079815","https://openalex.org/W3161186206","https://openalex.org/W3169040744","https://openalex.org/W3172226154","https://openalex.org/W3173913247","https://openalex.org/W3177993093","https://openalex.org/W3181669971","https://openalex.org/W3181848295","https://openalex.org/W3186851634","https://openalex.org/W3188856100","https://openalex.org/W3196715347","https://openalex.org/W3200354556","https://openalex.org/W3209381781","https://openalex.org/W4200452217","https://openalex.org/W4206091794","https://openalex.org/W4280597739","https://openalex.org/W4296706044","https://openalex.org/W4385278275","https://openalex.org/W6687681856","https://openalex.org/W6703271639"],"related_works":["https://openalex.org/W2000785801","https://openalex.org/W986318368","https://openalex.org/W2384410913","https://openalex.org/W2352878646","https://openalex.org/W2004734601","https://openalex.org/W2130149817","https://openalex.org/W2990194547","https://openalex.org/W1480123525","https://openalex.org/W2620865396","https://openalex.org/W3204654320"],"abstract_inverted_index":{"Peer-to-peer":[0],"(P2P)":[1],"energy":[2,69,178],"trading":[3,30,70,80,195],"among":[4,82],"multi-microgrids":[5],"has":[6],"emerged":[7],"as":[8],"a":[9,67,89,110,143,164],"promising":[10],"paradigm":[11,71],"to":[12,77,97,124,131,157],"facilitate":[13],"more":[14],"efficient":[15],"supply-demand":[16],"balancing":[17],"within":[18],"local":[19,128],"areas.":[20],"However,":[21],"existing":[22,38,215],"works":[23],"still":[24],"exhibit":[25],"limitations":[26],"in":[27,127],"terms":[28],"of":[29,53,93,166,182,193,202],"architecture":[31],"and":[32,55,106,137,150,175,222],"pricing":[33,120],"schemes.":[34],"In":[35,62],"addition,":[36],"the":[37,51,79,94,99,159,194,199,214],"multi-agent":[39,146],"deep":[40,148,151],"reinforcement":[41],"learning":[42],"(MADRL)":[43],"methods":[44,216],"suffer":[45],"from":[46],"computational":[47,139,220],"overload":[48],"caused":[49],"by":[50],"exploration":[52],"joint":[54],"hybrid":[56,75,133,145],"action":[57,135],"space":[58,136],"during":[59],"centralized":[60],"training.":[61],"this":[63],"paper,":[64],"we":[65,87,141],"propose":[66,142],"P2P":[68,111],"based":[72],"on":[73],"hierarchical":[74,91,144],"MADRL":[76],"maximize":[78],"profits":[81,196,211],"multiple":[83],"heterogeneous":[84],"MGs.":[85],"First,":[86],"design":[88],"novel":[90],"structure":[92],"MC":[95],"agent":[96],"model":[98],"coupled":[100],"interaction":[101],"between":[102],"flexible":[103,172],"demands":[104,173],"scheduling":[105,174],"autonomous":[107],"quotation.":[108],"Then,":[109],"market":[112],"that":[113,186,206],"employs":[114],"an":[115],"improved":[116],"mid-market":[117],"rate":[118],"(IMMR)":[119],"scheme":[121],"is":[122],"proposed":[123],"incentivize":[125],"participation":[126],"trading.":[129,179],"Furthermore,":[130],"handle":[132],"discrete-continuous":[134],"reduce":[138],"complexity,":[140],"double":[147],"Q-network":[149],"deterministic":[152],"policy":[153,161],"gradient":[154],"(hh-MADDQN-DDPG)":[155],"algorithm":[156],"split":[158],"optimal":[160],"learning-workload":[162],"into":[163],"sequence":[165],"two":[167],"sub-tasks.":[168],"The":[169],"DDQN":[170],"for":[171,177],"DDPG":[176],"Numerical":[180],"results":[181],"simulation":[183,203],"I":[184],"demonstrate":[185],"our":[187,207],"hh-MADDQN-DDPG":[188,208],"with":[189,213],"IMMR":[190],"increases":[191],"25%":[192],"averaged":[197],"over":[198],"baselines.":[200],"Results":[201],"II":[204],"show":[205],"provides":[209],"higher":[210],"compared":[212],"while":[217],"maintaining":[218],"better":[219],"performance":[221],"scalability.":[223]},"counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":33},{"year":2024,"cited_by_count":24},{"year":2023,"cited_by_count":5}],"updated_date":"2026-04-25T08:17:42.794288","created_date":"2025-10-10T00:00:00"}