{"id":"https://openalex.org/W4414930096","doi":"https://doi.org/10.48550/arxiv.2507.22174","title":"Spatial-Temporal Reinforcement Learning for Network Routing with Non-Markovian Traffic","display_name":"Spatial-Temporal Reinforcement Learning for Network Routing with Non-Markovian Traffic","publication_year":2025,"publication_date":"2025-07-27","ids":{"openalex":"https://openalex.org/W4414930096","doi":"https://doi.org/10.48550/arxiv.2507.22174"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2507.22174","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2507.22174","pdf_url":"https://arxiv.org/pdf/2507.22174","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2507.22174","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072367068","display_name":"Molly Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Wang, Molly","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5020917506","display_name":"Kin K. Leung","orcid":"https://orcid.org/0000-0002-3860-6257"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Leung, Kin. K","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5072367068"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10249","display_name":"Distributed Control Multi-Agent Systems","score":0.892300009727478,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10249","display_name":"Distributed Control Multi-Agent Systems","score":0.892300009727478,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10080","display_name":"Energy Efficient Wireless Sensor Networks","score":0.8417999744415283,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10148","display_name":"Advanced MIMO Systems Optimization","score":0.8226000070571899,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6730999946594238},{"id":"https://openalex.org/keywords/routing","display_name":"Routing (electronic design automation)","score":0.5076000094413757},{"id":"https://openalex.org/keywords/network-packet","display_name":"Network packet","score":0.4415999948978424},{"id":"https://openalex.org/keywords/static-routing","display_name":"Static routing","score":0.42890000343322754},{"id":"https://openalex.org/keywords/routing-domain","display_name":"Routing domain","score":0.4153999984264374},{"id":"https://openalex.org/keywords/routing-table","display_name":"Routing table","score":0.41130000352859497},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.38109999895095825},{"id":"https://openalex.org/keywords/link-state-routing-protocol","display_name":"Link-state routing protocol","score":0.37560001015663147},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.3675999939441681},{"id":"https://openalex.org/keywords/source-routing","display_name":"Source routing","score":0.3449000120162964}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7634000182151794},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6730999946594238},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.5181999802589417},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.5076000094413757},{"id":"https://openalex.org/C158379750","wikidata":"https://www.wikidata.org/wiki/Q214111","display_name":"Network packet","level":2,"score":0.4415999948978424},{"id":"https://openalex.org/C204948658","wikidata":"https://www.wikidata.org/wiki/Q1119410","display_name":"Static routing","level":4,"score":0.42890000343322754},{"id":"https://openalex.org/C94600068","wikidata":"https://www.wikidata.org/wiki/Q7371621","display_name":"Routing domain","level":5,"score":0.4153999984264374},{"id":"https://openalex.org/C184896649","wikidata":"https://www.wikidata.org/wiki/Q290066","display_name":"Routing table","level":4,"score":0.41130000352859497},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3919000029563904},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.38109999895095825},{"id":"https://openalex.org/C89305328","wikidata":"https://www.wikidata.org/wiki/Q1755411","display_name":"Link-state routing protocol","level":4,"score":0.37560001015663147},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3675999939441681},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3492000102996826},{"id":"https://openalex.org/C44010500","wikidata":"https://www.wikidata.org/wiki/Q1422567","display_name":"Source routing","level":5,"score":0.3449000120162964},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.33799999952316284},{"id":"https://openalex.org/C104954878","wikidata":"https://www.wikidata.org/wiki/Q1648707","display_name":"Routing protocol","level":3,"score":0.3312000036239624},{"id":"https://openalex.org/C174809319","wikidata":"https://www.wikidata.org/wiki/Q5973191","display_name":"IP forwarding","level":5,"score":0.328900009393692},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.3222000002861023},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.3059000074863434},{"id":"https://openalex.org/C115443555","wikidata":"https://www.wikidata.org/wiki/Q5367790","display_name":"Equal-cost multi-path routing","level":5,"score":0.30329999327659607},{"id":"https://openalex.org/C176715033","wikidata":"https://www.wikidata.org/wiki/Q2080768","display_name":"Traffic generation model","level":2,"score":0.301800012588501},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.29420000314712524},{"id":"https://openalex.org/C201100257","wikidata":"https://www.wikidata.org/wiki/Q393287","display_name":"Network traffic control","level":3,"score":0.2928999960422516},{"id":"https://openalex.org/C196423136","wikidata":"https://www.wikidata.org/wiki/Q7209671","display_name":"Policy-based routing","level":5,"score":0.29260000586509705},{"id":"https://openalex.org/C54108766","wikidata":"https://www.wikidata.org/wiki/Q391064","display_name":"Packet loss","level":3,"score":0.2906000018119812},{"id":"https://openalex.org/C2985963534","wikidata":"https://www.wikidata.org/wiki/Q7603704","display_name":"State information","level":3,"score":0.2824000120162964},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2766999900341034},{"id":"https://openalex.org/C133264484","wikidata":"https://www.wikidata.org/wiki/Q6056085","display_name":"Internet traffic engineering","level":4,"score":0.27219998836517334},{"id":"https://openalex.org/C50558702","wikidata":"https://www.wikidata.org/wiki/Q5535067","display_name":"Geographic routing","level":5,"score":0.27090001106262207},{"id":"https://openalex.org/C177818476","wikidata":"https://www.wikidata.org/wiki/Q12878103","display_name":"Hierarchical routing","level":5,"score":0.2651999890804291},{"id":"https://openalex.org/C188116033","wikidata":"https://www.wikidata.org/wiki/Q2664563","display_name":"Q-learning","level":3,"score":0.2639000117778778},{"id":"https://openalex.org/C9659607","wikidata":"https://www.wikidata.org/wiki/Q1268903","display_name":"Dynamic Source Routing","level":4,"score":0.2587999999523163},{"id":"https://openalex.org/C192126672","wikidata":"https://www.wikidata.org/wiki/Q1068715","display_name":"Telecommunications network","level":2,"score":0.2574999928474426},{"id":"https://openalex.org/C113508815","wikidata":"https://www.wikidata.org/wiki/Q193446","display_name":"Packet switching","level":3,"score":0.2508000135421753}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2507.22174","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2507.22174","pdf_url":"https://arxiv.org/pdf/2507.22174","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2507.22174","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2507.22174","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2507.22174","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2507.22174","pdf_url":"https://arxiv.org/pdf/2507.22174","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"Learning":[1],"(RL)":[2],"has":[3],"been":[4],"widely":[5],"used":[6],"for":[7,30,86,102],"packet":[8,87],"routing":[9,43],"in":[10,64,107],"communication":[11],"networks,":[12,56],"but":[13],"traditional":[14,92],"RL":[15,48,83],"methods":[16],"rely":[17],"on":[18],"the":[19,23,61],"Markov":[20],"assumption":[21],"that":[22,57],"current":[24],"state":[25],"contains":[26],"all":[27],"necessary":[28],"information":[29],"decision-making.":[31],"In":[32],"reality,":[33],"internet":[34],"traffic":[35,78],"is":[36],"non-Markovian,":[37],"and":[38,79,100],"past":[39],"states":[40],"do":[41,58],"influence":[42],"performance.":[44],"Moreover,":[45],"common":[46],"deep":[47],"approaches":[49],"use":[50],"function":[51],"approximators,":[52],"such":[53],"as":[54],"neural":[55],"not":[59],"model":[60],"spatial":[62],"structure":[63],"network":[65,74,108],"topologies.":[66],"To":[67],"address":[68],"these":[69],"shortcomings,":[70],"we":[71],"design":[72],"a":[73,81,105],"environment":[75],"with":[76],"non-Markovian":[77],"introduce":[80],"spatial-temporal":[82],"(STRL)":[84],"framework":[85],"routing.":[88],"Our":[89],"approach":[90],"outperforms":[91],"baselines":[93],"by":[94],"more":[95],"than":[96],"19%":[97],"during":[98],"training":[99],"7%":[101],"inference":[103],"despite":[104],"change":[106],"topology.":[109]},"counts_by_year":[],"updated_date":"2026-03-07T16:01:11.037858","created_date":"2025-10-10T00:00:00"}