{"id":"https://openalex.org/W4400111474","doi":"https://doi.org/10.1109/twc.2024.3416437","title":"Structured Reinforcement Learning for Delay-Optimal Data Transmission in Dense mmWave Networks","display_name":"Structured Reinforcement Learning for Delay-Optimal Data Transmission in Dense mmWave Networks","publication_year":2024,"publication_date":"2024-06-28","ids":{"openalex":"https://openalex.org/W4400111474","doi":"https://doi.org/10.1109/twc.2024.3416437"},"language":"en","primary_location":{"id":"doi:10.1109/twc.2024.3416437","is_oa":false,"landing_page_url":"https://doi.org/10.1109/twc.2024.3416437","pdf_url":null,"source":{"id":"https://openalex.org/S63459445","display_name":"IEEE Transactions on Wireless Communications","issn_l":"1536-1276","issn":["1536-1276","1558-2248"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Wireless Communications","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041888295","display_name":"Shufan Wang","orcid":"https://orcid.org/0000-0002-3919-0580"},"institutions":[{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shufan Wang","raw_affiliation_strings":["Department of Applied Mathematics, Statistics, and Computer Science, Stony Brook University, Stony Brook, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Applied Mathematics, Statistics, and Computer Science, Stony Brook University, Stony Brook, NY, USA","institution_ids":["https://openalex.org/I59553526"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102620407","display_name":"Guojun Xiong","orcid":null},"institutions":[{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Guojun Xiong","raw_affiliation_strings":["Department of Applied Mathematics, Statistics, and Computer Science, Stony Brook University, Stony Brook, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Applied Mathematics, Statistics, and Computer Science, Stony Brook University, Stony Brook, NY, USA","institution_ids":["https://openalex.org/I59553526"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053908209","display_name":"Shichen Zhang","orcid":"https://orcid.org/0000-0001-8432-0834"},"institutions":[{"id":"https://openalex.org/I87216513","display_name":"Michigan State University","ror":"https://ror.org/05hs6h993","country_code":"US","type":"education","lineage":["https://openalex.org/I87216513"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shichen Zhang","raw_affiliation_strings":["Department of Computer Science and Engineering, Michigan State University, East Lansing, MI, USA"],"raw_orcid":"https://orcid.org/0000-0001-8432-0834","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Michigan State University, East Lansing, MI, USA","institution_ids":["https://openalex.org/I87216513"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027120851","display_name":"Huacheng Zeng","orcid":"https://orcid.org/0000-0002-3272-5239"},"institutions":[{"id":"https://openalex.org/I87216513","display_name":"Michigan State University","ror":"https://ror.org/05hs6h993","country_code":"US","type":"education","lineage":["https://openalex.org/I87216513"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Huacheng Zeng","raw_affiliation_strings":["Department of Computer Science and Engineering, Michigan State University, East Lansing, MI, USA"],"raw_orcid":"https://orcid.org/0000-0002-3272-5239","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Michigan State University, East Lansing, MI, USA","institution_ids":["https://openalex.org/I87216513"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100402612","display_name":"Jian Li","orcid":"https://orcid.org/0000-0003-3642-3569"},"institutions":[{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jian Li","raw_affiliation_strings":["Department of Applied Mathematics, Statistics, and Computer Science, Stony Brook University, Stony Brook, NY, USA"],"raw_orcid":"https://orcid.org/0000-0003-3642-3569","affiliations":[{"raw_affiliation_string":"Department of Applied Mathematics, Statistics, and Computer Science, Stony Brook University, Stony Brook, NY, USA","institution_ids":["https://openalex.org/I59553526"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081489562","display_name":"Shivendra S. Panwar","orcid":"https://orcid.org/0000-0002-9822-6838"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shivendra S. Panwar","raw_affiliation_strings":["Department of Electrical and Computer Engineering, New York University, Brooklyn, NY, USA"],"raw_orcid":"https://orcid.org/0000-0002-9822-6838","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, New York University, Brooklyn, NY, USA","institution_ids":["https://openalex.org/I57206974"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.1853,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.47395215,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"23","issue":"10","first_page":"14546","last_page":"14559"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10936","display_name":"Millimeter-Wave Propagation and Modeling","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10936","display_name":"Millimeter-Wave Propagation and Modeling","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10148","display_name":"Advanced MIMO Systems Optimization","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11932","display_name":"Wireless Body Area Networks","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7590742111206055},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.755756139755249},{"id":"https://openalex.org/keywords/transmission","display_name":"Transmission (telecommunications)","score":0.5289419293403625},{"id":"https://openalex.org/keywords/transmission-delay","display_name":"Transmission delay","score":0.5034794211387634},{"id":"https://openalex.org/keywords/data-transmission","display_name":"Data transmission","score":0.4773198068141937},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.4740861654281616},{"id":"https://openalex.org/keywords/wireless","display_name":"Wireless","score":0.44015175104141235},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2924908995628357},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.27551621198654175}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7590742111206055},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.755756139755249},{"id":"https://openalex.org/C761482","wikidata":"https://www.wikidata.org/wiki/Q118093","display_name":"Transmission (telecommunications)","level":2,"score":0.5289419293403625},{"id":"https://openalex.org/C108921912","wikidata":"https://www.wikidata.org/wiki/Q7834639","display_name":"Transmission delay","level":3,"score":0.5034794211387634},{"id":"https://openalex.org/C557945733","wikidata":"https://www.wikidata.org/wiki/Q389772","display_name":"Data transmission","level":2,"score":0.4773198068141937},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.4740861654281616},{"id":"https://openalex.org/C555944384","wikidata":"https://www.wikidata.org/wiki/Q249","display_name":"Wireless","level":2,"score":0.44015175104141235},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2924908995628357},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.27551621198654175}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/twc.2024.3416437","is_oa":false,"landing_page_url":"https://doi.org/10.1109/twc.2024.3416437","pdf_url":null,"source":{"id":"https://openalex.org/S63459445","display_name":"IEEE Transactions on Wireless Communications","issn_l":"1536-1276","issn":["1536-1276","1558-2248"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Wireless Communications","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1147080076","display_name":null,"funder_award_id":"2148309, 2315614","funder_id":"https://openalex.org/F4320337388","funder_display_name":"Division of Computer and Network Systems"},{"id":"https://openalex.org/G1271705906","display_name":null,"funder_award_id":"W911NF-23-1-0072","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G8961849392","display_name":null,"funder_award_id":"DE-EE0009341","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"}],"funders":[{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320337388","display_name":"Division of Computer and Network Systems","ror":"https://ror.org/02rdzmk74"},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":64,"referenced_works":["https://openalex.org/W1662803991","https://openalex.org/W1984114931","https://openalex.org/W2013702609","https://openalex.org/W2044502527","https://openalex.org/W2054692642","https://openalex.org/W2056921512","https://openalex.org/W2065043565","https://openalex.org/W2076680677","https://openalex.org/W2087294514","https://openalex.org/W2115138651","https://openalex.org/W2116334496","https://openalex.org/W2122011919","https://openalex.org/W2135063910","https://openalex.org/W2139819161","https://openalex.org/W2149943599","https://openalex.org/W2167264214","https://openalex.org/W2256188983","https://openalex.org/W2343448572","https://openalex.org/W2511028367","https://openalex.org/W2606107937","https://openalex.org/W2613625031","https://openalex.org/W2711678697","https://openalex.org/W2761497348","https://openalex.org/W2794164469","https://openalex.org/W2807133510","https://openalex.org/W2885410086","https://openalex.org/W2886734605","https://openalex.org/W2962938719","https://openalex.org/W2962958278","https://openalex.org/W2990738158","https://openalex.org/W2997055691","https://openalex.org/W2997530752","https://openalex.org/W3010855363","https://openalex.org/W3022217175","https://openalex.org/W3044544925","https://openalex.org/W3099485648","https://openalex.org/W3103744592","https://openalex.org/W3104937890","https://openalex.org/W3105101694","https://openalex.org/W3106302305","https://openalex.org/W3128062393","https://openalex.org/W3136434144","https://openalex.org/W3171374485","https://openalex.org/W3175748441","https://openalex.org/W3184450435","https://openalex.org/W3185554818","https://openalex.org/W3207401138","https://openalex.org/W4206497039","https://openalex.org/W4213251304","https://openalex.org/W4226304595","https://openalex.org/W4226524168","https://openalex.org/W4283209274","https://openalex.org/W4283799802","https://openalex.org/W4297882551","https://openalex.org/W4301094597","https://openalex.org/W4312749179","https://openalex.org/W4388740203","https://openalex.org/W6637077600","https://openalex.org/W6740336040","https://openalex.org/W6743825324","https://openalex.org/W6769870811","https://openalex.org/W6784620602","https://openalex.org/W6798826543","https://openalex.org/W6863844807"],"related_works":["https://openalex.org/W4306904969","https://openalex.org/W2138720691","https://openalex.org/W4387163319","https://openalex.org/W2359247450","https://openalex.org/W2370180015","https://openalex.org/W1975553214","https://openalex.org/W2360932317","https://openalex.org/W2085512692","https://openalex.org/W1997631074","https://openalex.org/W3118170786"],"abstract_inverted_index":{"We":[0],"study":[1],"the":[2,44,48,83,124,198],"data":[3,20,33,210],"packet":[4,21],"transmission":[5],"problem":[6,65,74],"(<monospace":[7,78],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[8,79,88,120,130,148,164,171,184,221],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">mmDPT</monospace>)":[9],"in":[10,47,128,212],"dense":[11,104],"cell-free":[12],"millimeter":[13],"wave":[14],"(mmWave)":[15],"networks,":[16],"i.e.,":[17],"users":[18,36],"sending":[19],"requests":[22],"to":[23,35,42,51,156,191],"access":[24],"points":[25],"(APs)":[26],"via":[27,37],"uplinks":[28],"and":[29,56,62,98,140,188],"APs":[30,61],"transmitting":[31],"requested":[32],"packets":[34],"downlinks.":[38],"Our":[39],"objective":[40],"is":[41,90,186],"minimize":[43],"average":[45],"delay":[46],"system":[49],"due":[50],"APs\u2019":[52],"limited":[53],"service":[54],"capacity":[55],"unreliable":[57],"wireless":[58],"channels":[59],"between":[60],"users.":[63],"This":[64],"can":[66],"be":[67],"formulated":[68],"as":[69,194],"a":[70,112,138,158],"restless":[71],"multi-armed":[72],"bandits":[73],"with":[75],"fairness":[76],"constraint":[77],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">RMAB-F</monospace>).":[80],"Since":[81],"finding":[82],"optimal":[84,143],"policy":[85,145,202],"for":[86,101,118,146,203],"<monospace":[87,119,129,147,163,183,220],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">RMAB-F</monospace>":[89],"intractable,":[91],"existing":[92,224],"learning":[93,115],"algorithms":[94],"are":[95],"computationally":[96],"expensive":[97],"not":[99],"suitable":[100],"practical":[102,192],"dynamic":[103],"mmWave":[105,214],"networks.":[106],"In":[107],"this":[108,153],"paper,":[109],"we":[110,135,151],"propose":[111],"structured":[113,159],"reinforcement":[114],"(RL)":[116],"solution":[117],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">mmDPT</monospace>":[121],"by":[122],"exploiting":[123],"inherent":[125],"structure":[126,154,199],"encoded":[127],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">RMAB-F</monospace>.":[131,149],"To":[132],"achieve":[133],"this,":[134],"first":[136],"design":[137],"low-complexity":[139],"provably":[141,167],"asymptotically":[142],"index":[144,201],"Then,":[150],"leverage":[152],"information":[155],"develop":[157],"RL":[160],"algorithm":[161],"called":[162],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">mmDPT-TS</monospace>,":[165],"which":[166],"achieves":[168],"an":[169],"<inline-formula":[170],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">":[172],"<tex-math":[173],"notation=\"LaTeX\">$\\tilde":[174],"{\\mathcal":[175],"{O}}(\\sqrt":[176],"{T})$":[177],"</tex-math></inline-formula>":[178],"Bayesian":[179],"regret.":[180],"More":[181],"importantly,":[182],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">mmDPT-TS</monospace>":[185,222],"computation-efficient":[187],"thus":[189],"amenable":[190],"implementation,":[193],"it":[195],"fully":[196],"exploits":[197],"of":[200,219],"making":[204],"decisions.":[205],"Extensive":[206],"emulation":[207],"based":[208],"on":[209],"collected":[211],"realistic":[213],"networks":[215],"demonstrate":[216],"significant":[217],"gains":[218],"over":[223],"approaches.":[225]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-06-12T08:23:45.883708","created_date":"2025-10-10T00:00:00"}