{"id":"https://openalex.org/W4404788270","doi":"https://doi.org/10.1109/jiot.2024.3506713","title":"HRL-Based Proactive Caching Scheme for Vehicle\u2013Edge\u2013Cloud Collaborative System Applications","display_name":"HRL-Based Proactive Caching Scheme for Vehicle\u2013Edge\u2013Cloud Collaborative System Applications","publication_year":2024,"publication_date":"2024-11-27","ids":{"openalex":"https://openalex.org/W4404788270","doi":"https://doi.org/10.1109/jiot.2024.3506713"},"language":"en","primary_location":{"id":"doi:10.1109/jiot.2024.3506713","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jiot.2024.3506713","pdf_url":null,"source":{"id":"https://openalex.org/S2480266640","display_name":"IEEE Internet of Things Journal","issn_l":"2327-4662","issn":["2327-4662","2372-2541"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Internet of Things Journal","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100460485","display_name":"Xiaowei Wang","orcid":"https://orcid.org/0000-0001-8920-2669"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaowei Wang","raw_affiliation_strings":["State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, College of Mechanical and Vehicle Engineering, Hunan University, Changsha, China","College of Mechanical and Vehicle Engineering, State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, Hunan University, Changsha"],"raw_orcid":"https://orcid.org/0000-0001-8920-2669","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, College of Mechanical and Vehicle Engineering, Hunan University, Changsha, China","institution_ids":["https://openalex.org/I16609230"]},{"raw_affiliation_string":"College of Mechanical and Vehicle Engineering, State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, Hunan University, Changsha","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102727315","display_name":"Wenjie Ouyang","orcid":"https://orcid.org/0000-0002-7161-5032"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenjie Ouyang","raw_affiliation_strings":["State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, College of Mechanical and Vehicle Engineering, Hunan University, Changsha, China","the College of Mechanical and Vehicle Engineering, State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, Hunan University, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-7161-5032","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, College of Mechanical and Vehicle Engineering, Hunan University, Changsha, China","institution_ids":["https://openalex.org/I16609230"]},{"raw_affiliation_string":"the College of Mechanical and Vehicle Engineering, State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, Hunan University, Changsha, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100421776","display_name":"Yang Li","orcid":"https://orcid.org/0000-0003-3109-0953"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yang Li","raw_affiliation_strings":["State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, College of Mechanical and Vehicle Engineering, Hunan University, Changsha, China","College of Mechanical and Vehicle Engineering, State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, Hunan University, Changsha"],"raw_orcid":"https://orcid.org/0000-0003-3109-0953","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, College of Mechanical and Vehicle Engineering, Hunan University, Changsha, China","institution_ids":["https://openalex.org/I16609230"]},{"raw_affiliation_string":"College of Mechanical and Vehicle Engineering, State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, Hunan University, Changsha","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Chenlong Yin","orcid":null},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chenlong Yin","raw_affiliation_strings":["State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, College of Mechanical and Vehicle Engineering, Hunan University, Changsha, China","the College of Mechanical and Vehicle Engineering, State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, Hunan University, Changsha, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, College of Mechanical and Vehicle Engineering, Hunan University, Changsha, China","institution_ids":["https://openalex.org/I16609230"]},{"raw_affiliation_string":"the College of Mechanical and Vehicle Engineering, State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, Hunan University, Changsha, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069453458","display_name":"Mingyang Chen","orcid":"https://orcid.org/0009-0001-3882-0878"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mingyang Chen","raw_affiliation_strings":["Department of Electrical and Electronic Engineering, University College London, London, U.K","University of College London, London, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Electronic Engineering, University College London, London, U.K","institution_ids":["https://openalex.org/I45129253"]},{"raw_affiliation_string":"University of College London, London, UK","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030791821","display_name":"Guifu Ma","orcid":"https://orcid.org/0000-0002-6081-444X"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guifu Ma","raw_affiliation_strings":["State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, College of Mechanical and Vehicle Engineering, Hunan University, Changsha, China","the College of Mechanical and Vehicle Engineering, State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, Hunan University, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-6081-444X","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, College of Mechanical and Vehicle Engineering, Hunan University, Changsha, China","institution_ids":["https://openalex.org/I16609230"]},{"raw_affiliation_string":"the College of Mechanical and Vehicle Engineering, State Key Laboratory of Advanced Design and Manufacturing Technology for Vehicle, Hunan University, Changsha, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101760892","display_name":"Yingbai Hu","orcid":"https://orcid.org/0000-0003-2452-3570"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yingbai Hu","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China","The Chinese University of Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0003-2452-3570","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.8523,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.76044686,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"12","issue":"7","first_page":"8231","last_page":"8246"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9710999727249146,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10761","display_name":"Vehicular Ad Hoc Networks (VANETs)","score":0.965499997138977,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8250752687454224},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.7559902667999268},{"id":"https://openalex.org/keywords/scheme","display_name":"Scheme (mathematics)","score":0.6685206294059753},{"id":"https://openalex.org/keywords/enhanced-data-rates-for-gsm-evolution","display_name":"Enhanced Data Rates for GSM Evolution","score":0.5983447432518005},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.5443075299263},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.4766160547733307},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.2511715292930603},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.1354476809501648}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8250752687454224},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.7559902667999268},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.6685206294059753},{"id":"https://openalex.org/C162307627","wikidata":"https://www.wikidata.org/wiki/Q204833","display_name":"Enhanced Data Rates for GSM Evolution","level":2,"score":0.5983447432518005},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5443075299263},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.4766160547733307},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2511715292930603},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.1354476809501648},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/jiot.2024.3506713","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jiot.2024.3506713","pdf_url":null,"source":{"id":"https://openalex.org/S2480266640","display_name":"IEEE Internet of Things Journal","issn_l":"2327-4662","issn":["2327-4662","2372-2541"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Internet of Things Journal","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2629358559","display_name":null,"funder_award_id":"2023YFB2504700","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G7992915085","display_name":null,"funder_award_id":"2022RC1033","funder_id":"https://openalex.org/F4320330206","funder_display_name":"Science and Technology Program of Hunan Province"}],"funders":[{"id":"https://openalex.org/F4320330206","display_name":"Science and Technology Program of Hunan Province","ror":null},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W2100755716","https://openalex.org/W2219888463","https://openalex.org/W2746553466","https://openalex.org/W2807706688","https://openalex.org/W2889540440","https://openalex.org/W2902215693","https://openalex.org/W2919839278","https://openalex.org/W2963360527","https://openalex.org/W2981618170","https://openalex.org/W2987229805","https://openalex.org/W3000114004","https://openalex.org/W3000732140","https://openalex.org/W3002844053","https://openalex.org/W3006426012","https://openalex.org/W3012066402","https://openalex.org/W3012372362","https://openalex.org/W3020344301","https://openalex.org/W3037117000","https://openalex.org/W3037713526","https://openalex.org/W3043320740","https://openalex.org/W3047152165","https://openalex.org/W3049535205","https://openalex.org/W3119625338","https://openalex.org/W3120678039","https://openalex.org/W3123742938","https://openalex.org/W3134649272","https://openalex.org/W3142849873","https://openalex.org/W3170708168","https://openalex.org/W3194143722","https://openalex.org/W3194813201","https://openalex.org/W3199230503","https://openalex.org/W3217741927","https://openalex.org/W4210875011","https://openalex.org/W4210924629","https://openalex.org/W4213148557","https://openalex.org/W4213188403","https://openalex.org/W4285217780","https://openalex.org/W4293795158","https://openalex.org/W4294663398","https://openalex.org/W4312624510","https://openalex.org/W4372311864","https://openalex.org/W4376873958","https://openalex.org/W4383336742","https://openalex.org/W4386918882","https://openalex.org/W4388262472","https://openalex.org/W4388673361","https://openalex.org/W4389495316","https://openalex.org/W4390969737","https://openalex.org/W6757677476","https://openalex.org/W7021103559"],"related_works":["https://openalex.org/W4244478748","https://openalex.org/W4223488648","https://openalex.org/W2134969820","https://openalex.org/W2251605416","https://openalex.org/W1997222214","https://openalex.org/W2560439919","https://openalex.org/W4389340727","https://openalex.org/W3150465815","https://openalex.org/W2802581102","https://openalex.org/W1522862521"],"abstract_inverted_index":{"Proactive":[0],"caching":[1,29,51,59,66,112,139],"scheme":[2,52],"proactively":[3],"caches":[4],"data":[5],"before":[6],"the":[7,16,21,58,65,77,84,90,110,116,122,127,135,152,161,176,181,185,195,200,207,214,226],"request":[8,119],"comes,":[9],"which":[10],"is":[11,172],"essential":[12],"for":[13,150,179],"optimizing":[14],"of":[15,23,138,216,233,241],"resource":[17],"utilization":[18],"and":[19,36,61,92,102,121,209,219,237],"ensuring":[20],"stability":[22],"cached":[24],"contents.":[25],"However,":[26],"current":[27],"proactive":[28,50],"methods":[30,107,228],"suffer":[31],"from":[32,154],"high":[33],"computational":[34],"complexity":[35],"often":[37],"fall":[38],"into":[39],"suboptimal":[40],"solutions.":[41],"This":[42],"study":[43],"proposes":[44],"a":[45,71,145,164,169],"hierarchical":[46],"reinforcement":[47],"learning":[48,148],"(HRL)-based":[49],"to":[53,56,63,75,108,157,174,212],"determine":[54],"whether":[55],"update":[57,64],"queue":[60],"how":[62],"queue.":[67],"Specifically,":[68],"we":[69,143,189],"build":[70],"bi-level":[72],"optimization":[73,94],"model":[74],"maximize":[76],"content":[78,85,117],"cache":[79,86,234,242],"hit":[80,235],"ratio":[81,236],"while":[82,198],"minimizing":[83],"cost.":[87,243],"We":[88,204],"formulate":[89],"upper-level":[91,177],"lower-level":[93,182,202],"problems":[95],"as":[96,134],"Markov":[97],"decision":[98,166],"process":[99],"(MDP)":[100],"models":[101],"employ":[103],"double":[104],"deep":[105],"Q-learning":[106],"solve":[109],"optimal":[111],"strategy.":[113],"In":[114,141],"particular,":[115],"history":[118],"features":[120],"user":[123],"preference":[124],"generated":[125],"by":[126,229],"collaborative":[128],"filtering-based":[129],"recommendation":[130],"system":[131],"are":[132],"used":[133],"state":[136],"inputs":[137],"decisions.":[140],"addition,":[142],"design":[144],"two-stage":[146],"curriculum":[147],"paradigm":[149],"training":[151,180],"agents":[153,192],"easy":[155,186],"tasks":[156],"hard":[158,196],"tasks.":[159],"At":[160],"first":[162],"stage,":[163],"greedy":[165],"tree":[167],"with":[168],"posterior":[170],"knowledge":[171],"designed":[173],"provide":[175],"decisions":[178],"agent":[183],"in":[184,194,231,239],"task.":[187],"Then,":[188],"train":[190],"all":[191],"together":[193],"task":[197],"freezing":[199],"trained":[201],"agent.":[203],"conduct":[205],"both":[206],"simulation":[208],"on-road":[210],"experiments":[211],"validate":[213],"effectiveness":[215],"our":[217,223],"method,":[218],"results":[220],"indicate":[221],"that":[222],"method":[224],"outperforms":[225],"baseline":[227],"2.2%\u201337.2%":[230],"terms":[232,240],"75.4%\u201377.6%":[238]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}