{"id":"https://openalex.org/W4394966935","doi":"https://doi.org/10.1109/tccn.2024.3391322","title":"Multi-Agent Reinforcement Learning-Based Joint Caching and Routing in Heterogeneous Networks","display_name":"Multi-Agent Reinforcement Learning-Based Joint Caching and Routing in Heterogeneous Networks","publication_year":2024,"publication_date":"2024-04-19","ids":{"openalex":"https://openalex.org/W4394966935","doi":"https://doi.org/10.1109/tccn.2024.3391322"},"language":"en","primary_location":{"id":"doi:10.1109/tccn.2024.3391322","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tccn.2024.3391322","pdf_url":null,"source":{"id":"https://openalex.org/S2484188435","display_name":"IEEE Transactions on Cognitive Communications and Networking","issn_l":"2332-7731","issn":["2332-7731","2372-2045"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cognitive Communications and Networking","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000229589","display_name":"Meiyi Yang","orcid":"https://orcid.org/0000-0003-4548-4945"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Meiyi Yang","raw_affiliation_strings":["National Engineering Research Center of Advanced Network Technologies, School of Electronic and Information Engineering, Beijing Jiaotong University, Beijing, China","School of Electronic and Information Engineering, National Engineering Research Center of Advanced Network Technologies, Beijing Jiaotong University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Engineering Research Center of Advanced Network Technologies, School of Electronic and Information Engineering, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]},{"raw_affiliation_string":"School of Electronic and Information Engineering, National Engineering Research Center of Advanced Network Technologies, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055818694","display_name":"Deyun Gao","orcid":"https://orcid.org/0000-0002-8641-4057"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Deyun Gao","raw_affiliation_strings":["National Engineering Research Center of Advanced Network Technologies, School of Electronic and Information Engineering, Beijing Jiaotong University, Beijing, China","School of Electronic and Information Engineering, National Engineering Research Center of Advanced Network Technologies, Beijing Jiaotong University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-8641-4057","affiliations":[{"raw_affiliation_string":"National Engineering Research Center of Advanced Network Technologies, School of Electronic and Information Engineering, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]},{"raw_affiliation_string":"School of Electronic and Information Engineering, National Engineering Research Center of Advanced Network Technologies, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073704569","display_name":"Chuan Heng Foh","orcid":"https://orcid.org/0000-0002-5716-1396"},"institutions":[{"id":"https://openalex.org/I28290843","display_name":"University of Surrey","ror":"https://ror.org/00ks66431","country_code":"GB","type":"education","lineage":["https://openalex.org/I28290843"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Chuan Heng Foh","raw_affiliation_strings":["Department of Electrical and Electronic Engineering, 5GIC & 6GIC, Institute for Communication Systems, University of Surrey, Surrey, U.K"],"raw_orcid":"https://orcid.org/0000-0002-5716-1396","affiliations":[{"raw_affiliation_string":"Department of Electrical and Electronic Engineering, 5GIC & 6GIC, Institute for Communication Systems, University of Surrey, Surrey, U.K","institution_ids":["https://openalex.org/I28290843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036159136","display_name":"Wei Quan","orcid":"https://orcid.org/0000-0001-7454-0905"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Quan","raw_affiliation_strings":["National Engineering Research Center of Advanced Network Technologies, School of Electronic and Information Engineering, Beijing Jiaotong University, Beijing, China","School of Electronic and Information Engineering, National Engineering Research Center of Advanced Network Technologies, Beijing Jiaotong University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-7454-0905","affiliations":[{"raw_affiliation_string":"National Engineering Research Center of Advanced Network Technologies, School of Electronic and Information Engineering, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]},{"raw_affiliation_string":"School of Electronic and Information Engineering, National Engineering Research Center of Advanced Network Technologies, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5035919267","display_name":"Victor C. M. Leung","orcid":"https://orcid.org/0000-0003-3529-2640"},"institutions":[{"id":"https://openalex.org/I141945490","display_name":"University of British Columbia","ror":"https://ror.org/03rmrcq20","country_code":"CA","type":"education","lineage":["https://openalex.org/I141945490"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Victor C. M. Leung","raw_affiliation_strings":["Department of Electrical and Computer Engineering, The University of British Columbia, Vancouver, BC, Canada"],"raw_orcid":"https://orcid.org/0000-0003-3529-2640","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, The University of British Columbia, Vancouver, BC, Canada","institution_ids":["https://openalex.org/I141945490"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.9772,"has_fulltext":false,"cited_by_count":14,"citation_normalized_percentile":{"value":0.94176313,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"10","issue":"5","first_page":"1959","last_page":"1974"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10796","display_name":"Cooperative Communication and Network Coding","score":0.9896000027656555,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11896","display_name":"Opportunistic and Delay-Tolerant Networks","score":0.9886999726295471,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7978850603103638},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7463607788085938},{"id":"https://openalex.org/keywords/joint","display_name":"Joint (building)","score":0.5665767192840576},{"id":"https://openalex.org/keywords/routing","display_name":"Routing (electronic design automation)","score":0.5607316493988037},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.523949146270752},{"id":"https://openalex.org/keywords/policy-based-routing","display_name":"Policy-based routing","score":0.4301387071609497},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.4137341380119324},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38289493322372437},{"id":"https://openalex.org/keywords/static-routing","display_name":"Static routing","score":0.35923680663108826},{"id":"https://openalex.org/keywords/routing-protocol","display_name":"Routing protocol","score":0.22233828902244568},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08477026224136353}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7978850603103638},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7463607788085938},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.5665767192840576},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.5607316493988037},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.523949146270752},{"id":"https://openalex.org/C196423136","wikidata":"https://www.wikidata.org/wiki/Q7209671","display_name":"Policy-based routing","level":5,"score":0.4301387071609497},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.4137341380119324},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38289493322372437},{"id":"https://openalex.org/C204948658","wikidata":"https://www.wikidata.org/wiki/Q1119410","display_name":"Static routing","level":4,"score":0.35923680663108826},{"id":"https://openalex.org/C104954878","wikidata":"https://www.wikidata.org/wiki/Q1648707","display_name":"Routing protocol","level":3,"score":0.22233828902244568},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08477026224136353},{"id":"https://openalex.org/C170154142","wikidata":"https://www.wikidata.org/wiki/Q150737","display_name":"Architectural engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tccn.2024.3391322","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tccn.2024.3391322","pdf_url":null,"source":{"id":"https://openalex.org/S2484188435","display_name":"IEEE Transactions on Cognitive Communications and Networking","issn_l":"2332-7731","issn":["2332-7731","2372-2045"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cognitive Communications and Networking","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G608193306","display_name":null,"funder_award_id":"2023JBGP001","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"},{"id":"https://openalex.org/G6666390355","display_name":null,"funder_award_id":"62394323","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G820288592","display_name":null,"funder_award_id":"62394325","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W1972738071","https://openalex.org/W1994384380","https://openalex.org/W2048124060","https://openalex.org/W2339094490","https://openalex.org/W2522626811","https://openalex.org/W2547875792","https://openalex.org/W2565419328","https://openalex.org/W2617309511","https://openalex.org/W2624186268","https://openalex.org/W2790866313","https://openalex.org/W2807161440","https://openalex.org/W2885551694","https://openalex.org/W2885769478","https://openalex.org/W2967531314","https://openalex.org/W2971229345","https://openalex.org/W2979180961","https://openalex.org/W3004685513","https://openalex.org/W3033563497","https://openalex.org/W3035221273","https://openalex.org/W3044299892","https://openalex.org/W3047266715","https://openalex.org/W3110643301","https://openalex.org/W3136705879","https://openalex.org/W3157504245","https://openalex.org/W3167969180","https://openalex.org/W3170007158","https://openalex.org/W3178503712","https://openalex.org/W3179752366","https://openalex.org/W3205260830","https://openalex.org/W3206972316","https://openalex.org/W3217741927","https://openalex.org/W4210575242","https://openalex.org/W4296473620","https://openalex.org/W4297095015","https://openalex.org/W4315783855","https://openalex.org/W4385627407","https://openalex.org/W4387871190","https://openalex.org/W6631190155","https://openalex.org/W6729448088","https://openalex.org/W6734678876","https://openalex.org/W6738796088"],"related_works":["https://openalex.org/W1997091564","https://openalex.org/W2220567392","https://openalex.org/W146128134","https://openalex.org/W2607223747","https://openalex.org/W4316021914","https://openalex.org/W2352609928","https://openalex.org/W2102655743","https://openalex.org/W2090180826","https://openalex.org/W2915872201","https://openalex.org/W2152381136"],"abstract_inverted_index":{"In":[0],"this":[1,83],"paper,":[2],"we":[3,85,134],"explore":[4],"the":[5,33,49,61,65,116,148,163],"problem":[6,52],"of":[7,27,63,108,170],"minimizing":[8],"transmission":[9],"cost":[10],"among":[11],"cooperative":[12,50],"nodes":[13],"by":[14,123,127],"jointly":[15],"optimizing":[16],"caching":[17,45,51,68,97,172],"and":[18,111,153],"routing":[19,35],"in":[20,174],"a":[21,38,54,88,105,140],"hybrid":[22],"network":[23],"with":[24,60,114,162],"vital":[25],"support":[26],"service":[28],"differentiation.":[29],"We":[30,47],"show":[31],"that":[32,160],"optimal":[34],"policy":[36,42],"is":[37,71,78,151],"route-to-least":[39],"cost-cache":[40],"(RLC)":[41],"for":[43,142],"fixed":[44],"policy.":[46],"formulate":[48],"as":[53,139],"multi-agent":[55,91],"Markov":[56],"decision":[57,125],"process":[58,121],"(MDP)":[59],"goal":[62],"maximizing":[64],"long-term":[66],"expected":[67],"reward,":[69],"which":[70,115],"NP-complete":[72],"even":[73],"when":[74,146,177],"assuming":[75,147],"users\u2019":[76],"demand":[77],"perfectly":[79],"known.":[80,155],"To":[81],"solve":[82],"problem,":[84],"propose":[86],"C-MAAC,":[87],"partially":[89],"decentralized":[90,112],"deep":[92],"reinforcement":[93],"learning":[94,101],"(MADRL)-based":[95],"collaborative":[96],"algorithm":[98],"employing":[99],"actor-critic":[100],"model.":[102],"C-MAAC":[103,166],"has":[104],"key":[106],"characteristic":[107],"centralized":[109],"training":[110,120],"execution,":[113],"challenge":[117],"from":[118],"unstable":[119],"caused":[122],"simultaneous":[124],"made":[126],"all":[128],"agents":[129],"can":[130],"be":[131],"addressed.":[132],"Furthermore,":[133],"develop":[135],"an":[136,168],"optimization":[137],"method":[138],"criterion":[141],"our":[143],"MADRL":[144],"framework":[145],"content":[149],"popularity":[150],"stationary":[152],"prior":[154,164],"Our":[156],"experimental":[157],"results":[158],"demonstrate":[159],"compared":[161],"art,":[165],"increases":[167],"average":[169],"21.7%":[171],"reward":[173],"dynamic":[175],"environment":[176],"user":[178],"request":[179],"traffic":[180],"changes":[181],"rapidly.":[182]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}