{"id":"https://openalex.org/W6913248712","doi":"https://doi.org/10.5555/3635637.3663179","title":"Decision Making in Non-Stationary Environments with Policy-Augmented Search","display_name":"Decision Making in Non-Stationary Environments with Policy-Augmented Search","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W6913248712","doi":"https://doi.org/10.5555/3635637.3663179"},"language":"en","primary_location":{"id":"pmh:oai:pure.tue.nl:openaire_cris_publications/91fe01b9-1638-4cd7-a7ff-9b180774d6f6","is_oa":true,"landing_page_url":"https://research.tue.nl/en/publications/91fe01b9-1638-4cd7-a7ff-9b180774d6f6","pdf_url":null,"source":{"id":"https://openalex.org/S4406922641","display_name":"TU/e Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},"type":"conference-paper","indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://research.tue.nl/en/publications/91fe01b9-1638-4cd7-a7ff-9b180774d6f6","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Pettet, Ava","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pettet, Ava","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zhang, Yunuo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yunuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Baiting, Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Baiting, Luo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wray, Kyle Hollins","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wray, Kyle Hollins","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Baier, Hendrik","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Baier, Hendrik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Laszka, Aron","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Laszka, Aron","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Dubey, Abhishek","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dubey, Abhishek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Mukhopadhyay, Ayan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mukhopadhyay, Ayan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8324000239372253,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8324000239372253,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.07349999994039536,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.021199999377131462,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/monte-carlo-tree-search","display_name":"Monte Carlo tree search","score":0.781000018119812},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6622999906539917},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.5748999714851379},{"id":"https://openalex.org/keywords/tree","display_name":"Tree (set theory)","score":0.49709999561309814},{"id":"https://openalex.org/keywords/decision-tree","display_name":"Decision tree","score":0.47920000553131104},{"id":"https://openalex.org/keywords/contrast","display_name":"Contrast (vision)","score":0.47099998593330383},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.4496000111103058},{"id":"https://openalex.org/keywords/thompson-sampling","display_name":"Thompson sampling","score":0.4424000084400177}],"concepts":[{"id":"https://openalex.org/C46149586","wikidata":"https://www.wikidata.org/wiki/Q11785332","display_name":"Monte Carlo tree search","level":3,"score":0.781000018119812},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7477999925613403},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6622999906539917},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6129999756813049},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5778999924659729},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.5748999714851379},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.49709999561309814},{"id":"https://openalex.org/C84525736","wikidata":"https://www.wikidata.org/wiki/Q831366","display_name":"Decision tree","level":2,"score":0.47920000553131104},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.47099998593330383},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.4496000111103058},{"id":"https://openalex.org/C73602740","wikidata":"https://www.wikidata.org/wiki/Q7795822","display_name":"Thompson sampling","level":3,"score":0.4424000084400177},{"id":"https://openalex.org/C19499675","wikidata":"https://www.wikidata.org/wiki/Q232207","display_name":"Monte Carlo method","level":2,"score":0.4074999988079071},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.39559999108314514},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.3804999887943268},{"id":"https://openalex.org/C52421305","wikidata":"https://www.wikidata.org/wiki/Q1151499","display_name":"Particle filter","level":3,"score":0.3343000113964081},{"id":"https://openalex.org/C2986087404","wikidata":"https://www.wikidata.org/wiki/Q15946010","display_name":"Online learning","level":2,"score":0.3163999915122986},{"id":"https://openalex.org/C176248197","wikidata":"https://www.wikidata.org/wiki/Q458526","display_name":"Probably approximately correct learning","level":4,"score":0.2727000117301941},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.2721000015735626},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.27160000801086426},{"id":"https://openalex.org/C28901747","wikidata":"https://www.wikidata.org/wiki/Q177571","display_name":"Decision theory","level":2,"score":0.26980000734329224},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.259799987077713},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.2517000138759613}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:pure.tue.nl:openaire_cris_publications/91fe01b9-1638-4cd7-a7ff-9b180774d6f6","is_oa":true,"landing_page_url":"https://research.tue.nl/en/publications/91fe01b9-1638-4cd7-a7ff-9b180774d6f6","pdf_url":null,"source":{"id":"https://openalex.org/S4406922641","display_name":"TU/e Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:pure.tue.nl:openaire/91fe01b9-1638-4cd7-a7ff-9b180774d6f6","is_oa":true,"landing_page_url":"https://research.tue.nl/files/332314088/p2417.pdf","pdf_url":"https://pure.tue.nl/ws/files/332314088/p2417.pdf","source":{"id":"https://openalex.org/S4406922641","display_name":"TU/e Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:zenodo.org:14187585","is_oa":true,"landing_page_url":"https://doi.org/10.5555/3635637.3663179","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"AAMAS 2024, International Conference on Autonomous Agents and Multiagent Systems 2024","raw_type":"info:eu-repo/semantics/conferencePaper"}],"best_oa_location":{"id":"pmh:oai:pure.tue.nl:openaire_cris_publications/91fe01b9-1638-4cd7-a7ff-9b180774d6f6","is_oa":true,"landing_page_url":"https://research.tue.nl/en/publications/91fe01b9-1638-4cd7-a7ff-9b180774d6f6","pdf_url":null,"source":{"id":"https://openalex.org/S4406922641","display_name":"TU/e Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},"sustainable_development_goals":[{"score":0.7166227102279663,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[{"id":"https://openalex.org/G3252212252","display_name":"CAREER: Robust Online Decision Procedures for Societal Scale CPS","funder_award_id":"2238815","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320332815","display_name":"Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320338294","display_name":"Air Force Research Laboratory","ror":"https://ror.org/02e2egq70"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Sequential":[0],"decision-making":[1],"under":[2,162,209],"uncertainty":[3],"is":[4,61],"present":[5],"in":[6,64,70,83],"many":[7],"important":[8],"problems.":[9],"Two":[10],"popular":[11],"approaches":[12,80],"for":[13],"tackling":[14],"such":[15,84],"problems":[16],"are":[17,121],"reinforcement":[18],"learning":[19],"and":[20,101,106,170,184,194],"online":[21,147],"search":[22,148],"(e.g.,":[23],"Monte":[24,132],"Carlo":[25,133],"tree":[26,134],"search).":[27],"While":[28],"the":[29,37,43,50,68,88,98,112,154,166,173],"former":[30],"learns":[31],"a":[32,46,180],"policy":[33,144],"by":[34],"interacting":[35],"with":[36,145,188,212],"environment":[38,51,69,99],"(typically":[39],"done":[40],"before":[41,93],"execution),":[42],"latter":[44],"uses":[45],"generative":[47],"model":[48,152],"of":[49,153],"to":[52],"sample":[53],"promising":[54],"action":[55,169],"trajectories":[56],"at":[57],"decision":[58],"time.":[59,78],"Decision-making":[60],"particularly":[62],"challenging":[63],"non-stationary":[65,210],"environments,":[66],"where":[67],"which":[71,137,163],"an":[72,142,146,150],"agent":[73],"operates":[74],"can":[75,115],"change":[76],"over":[77],"Both":[79],"have":[81],"shortcomings":[82],"settings":[85,211],"--":[86],"on":[87,111,123,198],"one":[89],"hand,":[90,114],"policies":[91],"learned":[92],"execution":[94],"become":[95],"stale":[96],"when":[97,119],"changes":[100],"relearning":[102],"takes":[103],"both":[104],"time":[105,214],"computational":[107],"effort.":[108],"Online":[109],"search,":[110],"other":[113],"return":[116],"sub-optimal":[117],"actions":[118],"there":[120],"limitations":[122],"allowed":[124],"runtime.":[125],"In":[126],"this":[127],"paper,":[128],"we":[129,206],"introduce":[130],"\\textit{Policy-Augmented":[131],"search}":[135],"(PA-MCTS),":[136],"combines":[138],"action-value":[139],"estimates":[140],"from":[141],"out-of-date":[143],"using":[149],"up-to-date":[151],"environment.":[155],"We":[156,182],"prove":[157],"theoretical":[158],"results":[159],"showing":[160],"conditions":[161],"PA-MCTS":[164,178,216],"selects":[165],"one-step":[167],"optimal":[168],"also":[171],"bound":[172],"error":[174],"accrued":[175],"while":[176],"following":[177],"as":[179],"policy.":[181],"compare":[183],"contrast":[185],"our":[186],"approach":[187],"AlphaZero,":[189],"another":[190],"hybrid":[191],"planning":[192],"approach,":[193],"Deep":[195],"Q":[196],"Learning":[197],"several":[199],"OpenAI":[200],"Gym":[201],"environments.":[202],"Through":[203],"extensive":[204],"experiments,":[205],"show":[207],"that":[208],"limited":[213],"constraints,":[215],"outperforms":[217],"these":[218],"baselines.":[219]},"counts_by_year":[],"updated_date":"2026-07-19T07:52:34.831488","created_date":"2025-10-10T00:00:00"}