{"id":"https://openalex.org/W4226154801","doi":"https://doi.org/10.1109/jstsp.2022.3144874","title":"Distributed Reinforcement Learning for Age of Information Minimization in Real-Time IoT Systems","display_name":"Distributed Reinforcement Learning for Age of Information Minimization in Real-Time IoT Systems","publication_year":2022,"publication_date":"2022-01-25","ids":{"openalex":"https://openalex.org/W4226154801","doi":"https://doi.org/10.1109/jstsp.2022.3144874"},"language":"en","primary_location":{"id":"doi:10.1109/jstsp.2022.3144874","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2022.3144874","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023071584","display_name":"Sihua Wang","orcid":"https://orcid.org/0000-0001-6729-1592"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]},{"id":"https://openalex.org/I4392021250","display_name":"State Key Laboratory of Networking and Switching Technology","ror":"https://ror.org/00qtv5q45","country_code":null,"type":"facility","lineage":["https://openalex.org/I139759216","https://openalex.org/I4392021250"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sihua Wang","raw_affiliation_strings":["State Key Laboratory of Networking and Switching Technology, China","Beijing Key Laboratory of Network System Architecture and Convergence, Beijing University of Posts and Telecommunications, Beijing, China","Beijing Laboratory of Advanced Information Network, China"],"raw_orcid":"https://orcid.org/0000-0001-6729-1592","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Networking and Switching Technology, China","institution_ids":["https://openalex.org/I4392021250"]},{"raw_affiliation_string":"Beijing Key Laboratory of Network System Architecture and Convergence, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]},{"raw_affiliation_string":"Beijing Laboratory of Advanced Information Network, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072241033","display_name":"Mingzhe Chen","orcid":"https://orcid.org/0000-0003-2570-703X"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mingzhe Chen","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Princeton University, Princeton, NJ, USA"],"raw_orcid":"https://orcid.org/0000-0003-2570-703X","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Princeton University, Princeton, NJ, USA","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066878475","display_name":"Zhaohui Yang","orcid":"https://orcid.org/0000-0002-4475-589X"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Zhaohui Yang","raw_affiliation_strings":["Department of Electronic and Electrical Engineering, University College London, London, U.K"],"raw_orcid":"https://orcid.org/0000-0002-4475-589X","affiliations":[{"raw_affiliation_string":"Department of Electronic and Electrical Engineering, University College London, London, U.K","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009078493","display_name":"Changchuan Yin","orcid":"https://orcid.org/0000-0001-8969-1431"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Changchuan Yin","raw_affiliation_strings":["Beijing Laboratory of Advanced Information Network, China","Beijing Key Laboratory of Network System Architecture and Convergence, Beijing University of Posts and Telecommunications, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8969-1431","affiliations":[{"raw_affiliation_string":"Beijing Laboratory of Advanced Information Network, China","institution_ids":[]},{"raw_affiliation_string":"Beijing Key Laboratory of Network System Architecture and Convergence, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024108653","display_name":"Walid Saad","orcid":"https://orcid.org/0000-0003-2247-2458"},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Walid Saad","raw_affiliation_strings":["Wireless@VT, Bradley Department of Electrical and Computer Engineering, Virginia Tech, Arlington, VA, USA"],"raw_orcid":"https://orcid.org/0000-0003-2247-2458","affiliations":[{"raw_affiliation_string":"Wireless@VT, Bradley Department of Electrical and Computer Engineering, Virginia Tech, Arlington, VA, USA","institution_ids":["https://openalex.org/I859038795"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009164482","display_name":"Shuguang Cui","orcid":"https://orcid.org/0000-0003-2608-775X"},"institutions":[{"id":"https://openalex.org/I4210099586","display_name":"Shenzhen Research Institute of Big Data","ror":"https://ror.org/00z1gwf89","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210099586"]},{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]},{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuguang Cui","raw_affiliation_strings":["School of Science and Engineering (SSE) and Future Network of Intelligence Institute (FNii), The Chinese University of Hong Kong, Shenzhen, China","Peng Cheng Laboratory, Shenzhen, China","Shenzhen Research Institute of Big Data, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0003-2608-775X","affiliations":[{"raw_affiliation_string":"School of Science and Engineering (SSE) and Future Network of Intelligence Institute (FNii), The Chinese University of Hong Kong, Shenzhen, China","institution_ids":["https://openalex.org/I4210116924"]},{"raw_affiliation_string":"Peng Cheng Laboratory, Shenzhen, China","institution_ids":["https://openalex.org/I4210136793"]},{"raw_affiliation_string":"Shenzhen Research Institute of Big Data, Shenzhen, China","institution_ids":["https://openalex.org/I4210099586"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5042307561","display_name":"H. Vincent Poor","orcid":"https://orcid.org/0000-0002-2062-131X"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"H. Vincent Poor","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Princeton University, Princeton, NJ, USA"],"raw_orcid":"https://orcid.org/0000-0002-2062-131X","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Princeton University, Princeton, NJ, USA","institution_ids":["https://openalex.org/I20089843"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.0029,"has_fulltext":false,"cited_by_count":47,"citation_normalized_percentile":{"value":0.95851826,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"16","issue":"3","first_page":"501","last_page":"515"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13553","display_name":"Age of Information Optimization","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13553","display_name":"Age of Information Optimization","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12079","display_name":"IoT Networks and Protocols","score":0.9646999835968018,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10300","display_name":"Congenital Heart Disease Studies","score":0.9377999901771545,"subfield":{"id":"https://openalex.org/subfields/2713","display_name":"Epidemiology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7078211903572083},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6073593497276306},{"id":"https://openalex.org/keywords/minification","display_name":"Minification","score":0.5737266540527344},{"id":"https://openalex.org/keywords/internet-of-things","display_name":"Internet of Things","score":0.5673860907554626},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4315926134586334},{"id":"https://openalex.org/keywords/real-time-computing","display_name":"Real-time computing","score":0.4061126112937927},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3400895595550537},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.16001573204994202},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.08630117774009705}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7078211903572083},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6073593497276306},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.5737266540527344},{"id":"https://openalex.org/C81860439","wikidata":"https://www.wikidata.org/wiki/Q251212","display_name":"Internet of Things","level":2,"score":0.5673860907554626},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4315926134586334},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.4061126112937927},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3400895595550537},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.16001573204994202},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.08630117774009705}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/jstsp.2022.3144874","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2022.3144874","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.9100000262260437,"id":"https://metadata.un.org/sdg/7","display_name":"Affordable and clean energy"}],"awards":[{"id":"https://openalex.org/G1533899332","display_name":null,"funder_award_id":"EP/T015985/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G1804718253","display_name":"\u8f6f\u4ef6\u5b9a\u4e49\u8702\u7a9d\u7f51\u4e2d\u7684\u52a8\u6001\u8d44\u6e90\u7ba1\u7406\u5173\u952e\u6280\u672f\u7814\u7a76","funder_award_id":"61671086","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1834021097","display_name":null,"funder_award_id":"B17007","funder_id":"https://openalex.org/F4320327912","funder_display_name":"Higher Education Discipline Innovation Project"},{"id":"https://openalex.org/G2090423401","display_name":null,"funder_award_id":"61629101","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3455110177","display_name":"\u7cbe\u51c6\u519c\u4e1a\u5bbd\u5e45\u7f51\u7edc\u9ad8\u901a\u91cf\u7ec4\u7ec7\u4e0e\u975e\u52a3\u5206\u5c42\u4f18\u5316\u5173\u952e\u6280\u672f\u7814\u7a76","funder_award_id":"61871041","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4440750459","display_name":"6G Mitola Radio: Cognitive Brain That Has Collective Intelligence","funder_award_id":"EP/T015985/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G5140464331","display_name":null,"funder_award_id":"N00014-19-1-2621","funder_id":"https://openalex.org/F4320333591","funder_display_name":"Multidisciplinary University Research Initiative"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320327912","display_name":"Higher Education Discipline Innovation Project","ror":null},{"id":"https://openalex.org/F4320333591","display_name":"Multidisciplinary University Research Initiative","ror":null},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W2150194065","https://openalex.org/W2165131254","https://openalex.org/W2906875877","https://openalex.org/W2939949661","https://openalex.org/W2951452709","https://openalex.org/W2963537617","https://openalex.org/W2966441493","https://openalex.org/W2979846753","https://openalex.org/W2979966746","https://openalex.org/W2982819838","https://openalex.org/W3004273799","https://openalex.org/W3004977268","https://openalex.org/W3009360357","https://openalex.org/W3020925646","https://openalex.org/W3043543211","https://openalex.org/W3044319949","https://openalex.org/W3046988891","https://openalex.org/W3090615085","https://openalex.org/W3101422709","https://openalex.org/W3101940057","https://openalex.org/W3113239520","https://openalex.org/W3118039346","https://openalex.org/W3128585235","https://openalex.org/W3134737640","https://openalex.org/W3158743991","https://openalex.org/W3172394904","https://openalex.org/W3191352059","https://openalex.org/W3203789000","https://openalex.org/W6757677476","https://openalex.org/W6864424756"],"related_works":["https://openalex.org/W4245926026","https://openalex.org/W4311097251","https://openalex.org/W2586548817","https://openalex.org/W4306904969","https://openalex.org/W2625093826","https://openalex.org/W4200598720","https://openalex.org/W2921026492","https://openalex.org/W4247463117","https://openalex.org/W2138720691","https://openalex.org/W4362501864"],"abstract_inverted_index":{"In":[0,27],"this":[1,137,197],"paper,":[2],"the":[3,7,28,43,46,62,66,83,108,112,116,120,125,145,152,157,164,167,186,209,222,233,236,245,265,270,282],"problem":[4,174,180],"of":[5,10,12,19,21,45,65,90,148,156,166,189,248,253,272],"minimizing":[6,244],"weighted":[8,187,246],"sum":[9,188,247,271],"age":[11],"information":[13,72,118,134],"(AoI)":[14],"and":[15,69,111,128,133,151,192,250,278,281,290,301],"total":[16,283],"energy":[17,129,193,251,284],"consumption":[18,252,285],"Internet":[20],"Things":[22],"(IoT)":[23],"devices":[24,91,99,121,218],"is":[25,140,175,183],"studied.":[26],"considered":[29],"model,":[30],"each":[31,52,149],"IoT":[32],"device":[33,53,150,153,237],"monitors":[34],"a":[35,74,88,201,295,302],"physical":[36,47,67,168],"process":[37,48,169],"that":[38,264],"follows":[39],"nonlinear":[40],"dynamics.":[41],"As":[42],"dynamics":[44,64,105,165],"vary":[49],"over":[50],"time,":[51],"should":[54],"find":[55,221],"an":[56,178],"optimal":[57,224],"sampling":[58,132,146,210,225,234,304],"frequency":[59],"to":[60,73,79,92,142,161,184,219,276,288,294],"sample":[61,102],"real-time":[63],"system":[68],"send":[70],"sampled":[71,95,117],"base":[75],"station":[76],"(BS).":[77],"Due":[78],"limited":[80],"wireless":[81],"resources,":[82],"BS":[84,113,158],"can":[85,100,240,268],"only":[86],"select":[87],"subset":[89],"transmit":[93],"their":[94,103,228],"information.":[96],"Thus,":[97],"edge":[98,217],"cooperatively":[101,220],"monitored":[104],"based":[106],"on":[107],"local":[109,230],"observations":[110],"will":[114],"collect":[115],"from":[119],"immediately,":[122],"hence":[123],"avoiding":[124],"additional":[126],"time":[127],"used":[130],"for":[131,208],"transmission.":[135],"To":[136,195],"end,":[138],"it":[139],"necessary":[141],"jointly":[143],"optimize":[144],"policy":[147,211,226],"selection":[154,238],"scheme":[155,239],"so":[159],"as":[160,177],"accurately":[162],"monitor":[163],"using":[170,227],"minimum":[171],"energy.":[172],"This":[173],"formulated":[176],"optimization":[179],"whose":[181],"goal":[182],"minimize":[185],"AoI":[190,249,273],"cost":[191],"consumption.":[194],"solve":[196],"problem,":[198],"we":[199],"propose":[200],"novel":[202],"distributed":[203],"reinforcement":[204],"learning":[205],"(RL)":[206],"approach":[207],"optimization.":[212],"The":[213],"proposed":[214,266],"algorithm":[215,267],"enables":[216],"global":[223],"own":[229],"observations.":[231],"Given":[232],"policy,":[235],"be":[241],"optimized":[242],"thus":[243],"all":[254],"devices.":[255],"Simulations":[256],"with":[257],"real":[258],"PM":[259],"2.5":[260],"pollution":[261],"data":[262],"show":[263],"reduce":[269],"by":[274,286],"up":[275,287],"17.8%":[277],"33.9%,":[279],"respectively,":[280,292],"13.2%":[289],"35.1%,":[291],"compared":[293],"conventional":[296],"deep":[297],"Q":[298],"network":[299],"method":[300],"uniform":[303],"policy.":[305]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":11},{"year":2023,"cited_by_count":14},{"year":2022,"cited_by_count":8}],"updated_date":"2026-08-04T08:18:43.703281","created_date":"2025-10-10T00:00:00"}