{"id":"https://openalex.org/W3200113042","doi":"https://doi.org/10.1109/access.2021.3113501","title":"Deep Reinforcement Learning for Energy-Efficient Multi-Channel Transmissions in 5G Cognitive HetNets: Centralized, Decentralized and Transfer Learning Based Solutions","display_name":"Deep Reinforcement Learning for Energy-Efficient Multi-Channel Transmissions in 5G Cognitive HetNets: Centralized, Decentralized and Transfer Learning Based Solutions","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3200113042","doi":"https://doi.org/10.1109/access.2021.3113501","mag":"3200113042"},"language":"en","primary_location":{"id":"doi:10.1109/access.2021.3113501","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3113501","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/09540685.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/09540685.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008414543","display_name":"Anastasios Giannopoulos","orcid":"https://orcid.org/0000-0002-8602-7401"},"institutions":[{"id":"https://openalex.org/I174458059","display_name":"National Technical University of Athens","ror":"https://ror.org/03cx6bg69","country_code":"GR","type":"education","lineage":["https://openalex.org/I174458059"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"Anastasios Giannopoulos","raw_affiliation_strings":["Electrical and Computer Engineering, National Technical University of Athens, Athens, CO 15780 Greece. (e-mail: angianno_8@hotmail.com)","Electrical and Computer Engineering, National Technical University of Athens, Athens, CO 15780 Greece"],"raw_orcid":"https://orcid.org/0000-0002-8602-7401","affiliations":[{"raw_affiliation_string":"Electrical and Computer Engineering, National Technical University of Athens, Athens, CO 15780 Greece. (e-mail: angianno_8@hotmail.com)","institution_ids":["https://openalex.org/I174458059"]},{"raw_affiliation_string":"Electrical and Computer Engineering, National Technical University of Athens, Athens, CO 15780 Greece","institution_ids":["https://openalex.org/I174458059"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030378252","display_name":"Sotirios Spantideas","orcid":"https://orcid.org/0000-0002-1457-4131"},"institutions":[{"id":"https://openalex.org/I174458059","display_name":"National Technical University of Athens","ror":"https://ror.org/03cx6bg69","country_code":"GR","type":"education","lineage":["https://openalex.org/I174458059"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"Sotirios Spantideas","raw_affiliation_strings":["Electrical and Computer Engineering, National Technical University of Athens, Athens, CO 15780 Greece"],"raw_orcid":"https://orcid.org/0000-0002-1457-4131","affiliations":[{"raw_affiliation_string":"Electrical and Computer Engineering, National Technical University of Athens, Athens, CO 15780 Greece","institution_ids":["https://openalex.org/I174458059"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040416278","display_name":"Nikolaos C. Kapsalis","orcid":"https://orcid.org/0000-0003-1084-2546"},"institutions":[{"id":"https://openalex.org/I174458059","display_name":"National Technical University of Athens","ror":"https://ror.org/03cx6bg69","country_code":"GR","type":"education","lineage":["https://openalex.org/I174458059"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"Nikolaos Kapsalis","raw_affiliation_strings":["Electrical and Computer Engineering, National Technical University of Athens, Athens, CO 15780 Greece"],"raw_orcid":"https://orcid.org/0000-0003-1084-2546","affiliations":[{"raw_affiliation_string":"Electrical and Computer Engineering, National Technical University of Athens, Athens, CO 15780 Greece","institution_ids":["https://openalex.org/I174458059"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011492909","display_name":"Panagiotis Karkazis","orcid":"https://orcid.org/0000-0003-4971-826X"},"institutions":[{"id":"https://openalex.org/I4210094138","display_name":"University of West Attica","ror":"https://ror.org/00r2r5k05","country_code":"GR","type":"education","lineage":["https://openalex.org/I4210094138"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"Panagiotis Karkazis","raw_affiliation_strings":["Department of Informatics and Computer Engineering, University of West Attica, 12243 Athens, Greece"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Informatics and Computer Engineering, University of West Attica, 12243 Athens, Greece","institution_ids":["https://openalex.org/I4210094138"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5039352682","display_name":"Panagiotis Trakadas","orcid":"https://orcid.org/0000-0002-5146-5954"},"institutions":[{"id":"https://openalex.org/I200777214","display_name":"National and Kapodistrian University of Athens","ror":"https://ror.org/04gnjpq42","country_code":"GR","type":"education","lineage":["https://openalex.org/I200777214"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"Panagiotis Trakadas","raw_affiliation_strings":["Department of Ports Management and Shipping, National and Kapodistrian University of Athens, Evia, 34400, Greece"],"raw_orcid":"https://orcid.org/0000-0002-5146-5954","affiliations":[{"raw_affiliation_string":"Department of Ports Management and Shipping, National and Kapodistrian University of Athens, Evia, 34400, Greece","institution_ids":["https://openalex.org/I200777214"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":5.2885,"has_fulltext":true,"cited_by_count":63,"citation_normalized_percentile":{"value":0.96310801,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"9","issue":null,"first_page":"129358","last_page":"129374"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10148","display_name":"Advanced MIMO Systems Optimization","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10148","display_name":"Advanced MIMO Systems Optimization","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11392","display_name":"Energy Harvesting in Wireless Networks","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11458","display_name":"Advanced Wireless Communication Technologies","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8255166411399841},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7473295331001282},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.515577495098114},{"id":"https://openalex.org/keywords/transmission","display_name":"Transmission (telecommunications)","score":0.4663257896900177},{"id":"https://openalex.org/keywords/channel","display_name":"Channel (broadcasting)","score":0.4618721306324005},{"id":"https://openalex.org/keywords/quality-of-service","display_name":"Quality of service","score":0.44440677762031555},{"id":"https://openalex.org/keywords/efficient-energy-use","display_name":"Efficient energy use","score":0.44373083114624023},{"id":"https://openalex.org/keywords/wireless","display_name":"Wireless","score":0.437056303024292},{"id":"https://openalex.org/keywords/maximum-power-transfer-theorem","display_name":"Maximum power transfer theorem","score":0.41809260845184326},{"id":"https://openalex.org/keywords/heterogeneous-network","display_name":"Heterogeneous network","score":0.4119400084018707},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.40966516733169556},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3630181849002838},{"id":"https://openalex.org/keywords/power","display_name":"Power (physics)","score":0.3548814058303833},{"id":"https://openalex.org/keywords/wireless-network","display_name":"Wireless network","score":0.33925145864486694},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.18517914414405823},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.13279250264167786}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8255166411399841},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7473295331001282},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.515577495098114},{"id":"https://openalex.org/C761482","wikidata":"https://www.wikidata.org/wiki/Q118093","display_name":"Transmission (telecommunications)","level":2,"score":0.4663257896900177},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.4618721306324005},{"id":"https://openalex.org/C5119721","wikidata":"https://www.wikidata.org/wiki/Q220501","display_name":"Quality of service","level":2,"score":0.44440677762031555},{"id":"https://openalex.org/C2742236","wikidata":"https://www.wikidata.org/wiki/Q924713","display_name":"Efficient energy use","level":2,"score":0.44373083114624023},{"id":"https://openalex.org/C555944384","wikidata":"https://www.wikidata.org/wiki/Q249","display_name":"Wireless","level":2,"score":0.437056303024292},{"id":"https://openalex.org/C67186554","wikidata":"https://www.wikidata.org/wiki/Q17103352","display_name":"Maximum power transfer theorem","level":3,"score":0.41809260845184326},{"id":"https://openalex.org/C158207573","wikidata":"https://www.wikidata.org/wiki/Q5747224","display_name":"Heterogeneous network","level":4,"score":0.4119400084018707},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.40966516733169556},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3630181849002838},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.3548814058303833},{"id":"https://openalex.org/C108037233","wikidata":"https://www.wikidata.org/wiki/Q11375","display_name":"Wireless network","level":3,"score":0.33925145864486694},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18517914414405823},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.13279250264167786},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2021.3113501","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3113501","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/09540685.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:6e53f76930c2411f899edc710684a96e","is_oa":true,"landing_page_url":"https://doaj.org/article/6e53f76930c2411f899edc710684a96e","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 9, Pp 129358-129374 (2021)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2021.3113501","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3113501","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/09540685.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.8899999856948853,"id":"https://metadata.un.org/sdg/7","display_name":"Affordable and clean energy"}],"awards":[{"id":"https://openalex.org/G1493237323","display_name":null,"funder_award_id":"H2020-ICT-2020-1","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G1535130713","display_name":null,"funder_award_id":"H2020-ICT","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G4456841783","display_name":null,"funder_award_id":"957317","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G4937468798","display_name":null,"funder_award_id":"H2020","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G7106928629","display_name":null,"funder_award_id":"Grant Agreement H2020-ICT-2020-1, number 957317","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G7842005466","display_name":null,"funder_award_id":"Horizon 2020","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320332999","display_name":"Horizon 2020 Framework Programme","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3200113042.pdf","grobid_xml":"https://content.openalex.org/works/W3200113042.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W41554520","https://openalex.org/W1995875735","https://openalex.org/W1997834106","https://openalex.org/W2098473710","https://openalex.org/W2121863487","https://openalex.org/W2128145324","https://openalex.org/W2136822103","https://openalex.org/W2327455150","https://openalex.org/W2561773934","https://openalex.org/W2591677992","https://openalex.org/W2599615286","https://openalex.org/W2610853759","https://openalex.org/W2612336410","https://openalex.org/W2738113351","https://openalex.org/W2741401130","https://openalex.org/W2774086490","https://openalex.org/W2937571395","https://openalex.org/W2954249867","https://openalex.org/W2955133642","https://openalex.org/W2963020121","https://openalex.org/W2968782695","https://openalex.org/W2970269258","https://openalex.org/W2973449430","https://openalex.org/W2994761817","https://openalex.org/W2998669109","https://openalex.org/W3005285703","https://openalex.org/W3015131984","https://openalex.org/W3093951152","https://openalex.org/W3096831136","https://openalex.org/W3098476189","https://openalex.org/W3127355490","https://openalex.org/W3176546244","https://openalex.org/W3182816986","https://openalex.org/W6797336534","https://openalex.org/W6798519237"],"related_works":["https://openalex.org/W4306904969","https://openalex.org/W2138720691","https://openalex.org/W4362501864","https://openalex.org/W4380318855","https://openalex.org/W3084456289","https://openalex.org/W1990395098","https://openalex.org/W2334642245","https://openalex.org/W2268092139","https://openalex.org/W2963527963","https://openalex.org/W2805060255"],"abstract_inverted_index":{"Energy":[0],"efficiency":[1],"(EE)":[2],"constitutes":[3],"a":[4,24,77,93,111,114,118,215],"key":[5],"target":[6],"in":[7,101,132],"the":[8,16,36,51,54,58,62,67,74,133,150,153,166,172,190,208,225,255,259,264],"deployment":[9],"of":[10,39,53,61,88,210,217,266],"5G":[11,41,134],"networks,":[12],"especially":[13],"due":[14],"to":[15,49,84,125,178],"increased":[17,232],"densification":[18],"and":[19,43,65,97,117,156,251,270],"heterogeneity.":[20],"In":[21,187],"this":[22],"paper,":[23],"Deep":[25],"Q-Network":[26],"(DQN)":[27],"based":[28],"power":[29,60,99,154,226],"control":[30],"scheme":[31,236],"is":[32,80,130,212],"proposed":[33],"for":[34,165,199,219,245],"improving":[35],"system-level":[37],"EE":[38,52,75,145,239,261],"two-tier":[40],"heterogeneous":[42],"multi-channel":[44],"cells.":[45],"The":[46],"algorithm":[47],"aims":[48],"maximize":[50],"system":[55],"by":[56],"regulating":[57],"transmission":[59,103],"downlink":[63],"channels":[64],"reconfiguring":[66],"user":[68,90],"association":[69],"scheme.":[70],"To":[71,105],"efficiently":[72],"solve":[73],"problem,":[76],"DQN-based":[78,109],"method":[79,122],"established,":[81],"properly":[82],"modified":[83],"ensure":[85],"adequate":[86],"QoS":[87],"each":[89,102,182],"(via":[91],"defining":[92],"demand-driven":[94],"rewarding":[95],"system)":[96],"near-optimal":[98],"adjustment":[100],"link.":[104],"directly":[106],"compare":[107],"different":[108],"approaches,":[110],"centralized":[112],"(C-DQN),":[113],"multi-agent":[115],"(MA-DQN)":[116],"transfer":[119],"learning-based":[120],"(T-DQN)":[121],"are":[123,228],"deployed":[124],"address":[126],"whether":[127],"their":[128],"applicability":[129],"beneficial":[131,197],"HetNets.":[135],"Results":[136],"confirmed":[137],"that":[138],"DQN-assisted":[139],"actions":[140],"could":[141],"offer":[142],"enhanced":[143],"network-wide":[144],"performance,":[146],"as":[147],"they":[148],"balance":[149],"trade-off":[151],"between":[152],"consumption":[155],"achieved":[157],"throughput":[158],"(in":[159],"Mbps/Watt).":[160],"Excessive":[161],"performance":[162],"was":[163],"observed":[164],"MA-DQN":[167,193,211,257],"approach":[168],"(>5":[169],"Mbps/Watt),":[170,243],"since":[171],"decentralized":[173],"learning":[174],"supports":[175],"low-dimensional":[176],"agents":[177],"be":[179],"coordinated":[180],"with":[181,248],"other":[183],"through":[184],"global":[185],"rewards.":[186],"further":[188],"comparing":[189],"T-DQN":[191,195,235],"against":[192],"solutions,":[194],"presents":[196],"usage":[198,209],"very":[200,203],"low":[201],"or":[202],"high":[204],"inter-cell":[205,221],"distances,":[206],"whereas":[207],"preferred":[213],"(by":[214],"factor":[216],"~1.3)":[218],"intermediate":[220],"distances":[222],"(100-600m),":[223],"where":[224],"savings":[227],"feasible":[229],"towards":[230],"achieving":[231],"EE.":[233],"Furthermore,":[234],"guarantees":[237],"good":[238],"solutions":[240,262],"(above":[241],"2":[242],"even":[244],"densely-deployed":[246],"macro-cells,":[247],"effortless":[249],"training":[250,268,272],"memory":[252],"requirements.":[253],"On":[254],"contrary,":[256],"offers":[258],"best":[260],"at":[263],"expense":[265],"massive":[267],"resources":[269],"required":[271],"time.":[273]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":13},{"year":2023,"cited_by_count":18},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":2}],"updated_date":"2026-06-16T09:24:06.705377","created_date":"2025-10-10T00:00:00"}