{"id":"https://openalex.org/W2962787188","doi":"https://doi.org/10.1109/lra.2018.2801479","title":"A Solution to Time-Varying Markov Decision Processes","display_name":"A Solution to Time-Varying Markov Decision Processes","publication_year":2018,"publication_date":"2018-02-02","ids":{"openalex":"https://openalex.org/W2962787188","doi":"https://doi.org/10.1109/lra.2018.2801479","mag":"2962787188"},"language":"en","primary_location":{"id":"doi:10.1109/lra.2018.2801479","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lra.2018.2801479","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101917996","display_name":"Lantao Liu","orcid":"https://orcid.org/0000-0002-6796-6817"},"institutions":[{"id":"https://openalex.org/I4210119109","display_name":"Indiana University Bloomington","ror":"https://ror.org/02k40bc56","country_code":"US","type":"education","lineage":["https://openalex.org/I4210119109","https://openalex.org/I592451"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Lantao Liu","raw_affiliation_strings":["Intelligent Systems Engineering Department, Indiana University, Bloomington, IN, USA"],"raw_orcid":"https://orcid.org/0000-0002-6796-6817","affiliations":[{"raw_affiliation_string":"Intelligent Systems Engineering Department, Indiana University, Bloomington, IN, USA","institution_ids":["https://openalex.org/I4210119109"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077367921","display_name":"Gaurav S. Sukhatme","orcid":"https://orcid.org/0000-0003-2408-474X"},"institutions":[{"id":"https://openalex.org/I4210119109","display_name":"Indiana University Bloomington","ror":"https://ror.org/02k40bc56","country_code":"US","type":"education","lineage":["https://openalex.org/I4210119109","https://openalex.org/I592451"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gaurav S. Sukhatme","raw_affiliation_strings":["Intelligent Systems Engineering Department, Indiana University, Bloomington, IN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Intelligent Systems Engineering Department, Indiana University, Bloomington, IN, USA","institution_ids":["https://openalex.org/I4210119109"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5101917996"],"corresponding_institution_ids":["https://openalex.org/I4210119109"],"apc_list":null,"apc_paid":null,"fwci":4.5691,"has_fulltext":false,"cited_by_count":43,"citation_normalized_percentile":{"value":0.95765993,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"3","issue":"3","first_page":"1631","last_page":"1638"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9911999702453613,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9911999702453613,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11622","display_name":"Maritime Navigation and Safety","score":0.9868999719619751,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10876","display_name":"Fault Detection and Control Systems","score":0.9731000065803528,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.8783073425292969},{"id":"https://openalex.org/keywords/dimension","display_name":"Dimension (graph theory)","score":0.6001260876655579},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5930091142654419},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.5320582985877991},{"id":"https://openalex.org/keywords/markov-chain","display_name":"Markov chain","score":0.5206462740898132},{"id":"https://openalex.org/keywords/state-space","display_name":"State space","score":0.5104328989982605},{"id":"https://openalex.org/keywords/robotics","display_name":"Robotics","score":0.5101678371429443},{"id":"https://openalex.org/keywords/time-horizon","display_name":"Time horizon","score":0.4671007990837097},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.46223244071006775},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.449924111366272},{"id":"https://openalex.org/keywords/partially-observable-markov-decision-process","display_name":"Partially observable Markov decision process","score":0.43953847885131836},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.436414897441864},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3988332748413086},{"id":"https://openalex.org/keywords/markov-model","display_name":"Markov model","score":0.366887629032135},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.3388529121875763},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.24647685885429382},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.18930602073669434},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.17820900678634644}],"concepts":[{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.8783073425292969},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.6001260876655579},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5930091142654419},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.5320582985877991},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.5206462740898132},{"id":"https://openalex.org/C72434380","wikidata":"https://www.wikidata.org/wiki/Q230930","display_name":"State space","level":2,"score":0.5104328989982605},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.5101678371429443},{"id":"https://openalex.org/C28761237","wikidata":"https://www.wikidata.org/wiki/Q7805321","display_name":"Time horizon","level":2,"score":0.4671007990837097},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.46223244071006775},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.449924111366272},{"id":"https://openalex.org/C17098449","wikidata":"https://www.wikidata.org/wiki/Q176814","display_name":"Partially observable Markov decision process","level":4,"score":0.43953847885131836},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.436414897441864},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3988332748413086},{"id":"https://openalex.org/C163836022","wikidata":"https://www.wikidata.org/wiki/Q6771326","display_name":"Markov model","level":3,"score":0.366887629032135},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3388529121875763},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.24647685885429382},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.18930602073669434},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.17820900678634644},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/lra.2018.2801479","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lra.2018.2801479","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"},{"id":"pmh:oai:iu.tind.io:1048","is_oa":false,"landing_page_url":"http://iu.tind.io/record/1048","pdf_url":null,"source":{"id":"https://openalex.org/S4306402656","display_name":"IUScholarWorks Open (Indiana University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I592451","host_organization_name":"Indiana University","host_organization_lineage":["https://openalex.org/I592451"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://iu.tind.io/record/1048","raw_type":"Text"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/14","score":0.75,"display_name":"Life below water"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W108082272","https://openalex.org/W1515851193","https://openalex.org/W1597303641","https://openalex.org/W1977655452","https://openalex.org/W2005308357","https://openalex.org/W2011985563","https://openalex.org/W2019306960","https://openalex.org/W2034656697","https://openalex.org/W2053589029","https://openalex.org/W2061504687","https://openalex.org/W2066430826","https://openalex.org/W2082878954","https://openalex.org/W2098432798","https://openalex.org/W2100986500","https://openalex.org/W2109910161","https://openalex.org/W2123157758","https://openalex.org/W2131600418","https://openalex.org/W2133993682","https://openalex.org/W2145483940","https://openalex.org/W2168945108","https://openalex.org/W2334782222","https://openalex.org/W2519757542","https://openalex.org/W2949887691","https://openalex.org/W3041202696","https://openalex.org/W4245108548","https://openalex.org/W6604443580","https://openalex.org/W6635767209","https://openalex.org/W6683073833","https://openalex.org/W6684839497","https://openalex.org/W6688843669"],"related_works":["https://openalex.org/W2096013579","https://openalex.org/W1589140671","https://openalex.org/W1760611253","https://openalex.org/W52153049","https://openalex.org/W2951545791","https://openalex.org/W1515117609","https://openalex.org/W2294884454","https://openalex.org/W4323315247","https://openalex.org/W3169161914","https://openalex.org/W3208638985"],"abstract_inverted_index":{"We":[0,103,128],"consider":[1],"a":[2,53,66,88,108,111,133],"decision-making":[3],"problem":[4],"where":[5],"the":[6,21,31,46,57,73,114,121,125],"environment":[7],"varies":[8],"both":[9],"in":[10,37,132],"space":[11],"and":[12,71,80,142],"time.":[13],"Such":[14],"problems":[15],"arise":[16],"naturally":[17],"when":[18],"considering,":[19],"e.g.,":[20],"navigation":[22,32,136],"of":[23,33,84,113],"an":[24,34],"underwater":[25],"robot":[26],"amidst":[27],"ocean":[28,140],"currents":[29],"or":[30],"aerial":[35],"vehicle":[36],"wind.":[38],"To":[39],"model":[40,70],"such":[41],"spatiotemporal":[42,139],"variation,":[43],"we":[44],"extend":[45],"standard":[47,74],"Markov":[48,59],"decision":[49,60],"process":[50,61],"(MDP)":[51],"to":[52,87,93,95,106],"new":[54],"framework":[55,89,131],"called":[56],"time-varying":[58,67,97],"(TVMDP).":[62],"The":[63],"TVMDP":[64,109],"has":[65],"state":[68,85],"transition":[69,98],"transforms":[72],"MDP":[75,115],"that":[76,90,144],"considers":[77],"only":[78],"immediate":[79],"static":[81],"uncertainty":[82],"descriptions":[83],"transitions,":[86],"is":[91],"able":[92],"adapt":[94],"future":[96],"dynamics":[99,123],"over":[100],"some":[101],"horizon.":[102],"show":[104,143],"how":[105],"solve":[107],"via":[110],"redesign":[112],"value":[116],"propagation":[117],"mechanisms":[118],"by":[119],"incorporating":[120],"introduced":[122],"along":[124],"temporal":[126],"dimension.":[127],"validate":[129],"our":[130],"marine":[134],"robotics":[135],"setting":[137],"using":[138],"data":[141],"it":[145],"outperforms":[146],"prior":[147],"efforts.":[148]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":11},{"year":2020,"cited_by_count":8},{"year":2019,"cited_by_count":7},{"year":2018,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}