{"id":"https://openalex.org/W4391022014","doi":"https://doi.org/10.1109/cdc49753.2023.10383559","title":"Model-Free Reinforcement Learning for Spatiotemporal Tasks Using Symbolic Automata","display_name":"Model-Free Reinforcement Learning for Spatiotemporal Tasks Using Symbolic Automata","publication_year":2023,"publication_date":"2023-12-13","ids":{"openalex":"https://openalex.org/W4391022014","doi":"https://doi.org/10.1109/cdc49753.2023.10383559"},"language":"en","primary_location":{"id":"doi:10.1109/cdc49753.2023.10383559","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cdc49753.2023.10383559","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 62nd IEEE Conference on Decision and Control (CDC)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044356220","display_name":"Anand Balakrishnan","orcid":"https://orcid.org/0000-0002-5778-3857"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Anand Balakrishnan","raw_affiliation_strings":["University of Southern California,Los Angeles,California,USA","University of Southern California, Los Angeles, California, USA"],"affiliations":[{"raw_affiliation_string":"University of Southern California,Los Angeles,California,USA","institution_ids":["https://openalex.org/I1174212"]},{"raw_affiliation_string":"University of Southern California, Los Angeles, California, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043437345","display_name":"Stefan Jak\u0161i\u0107","orcid":"https://orcid.org/0000-0002-3203-9415"},"institutions":[{"id":"https://openalex.org/I132118926","display_name":"Austrian Institute of Technology","ror":"https://ror.org/04knbh022","country_code":"AT","type":"facility","lineage":["https://openalex.org/I132118926"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Stefan Jak\u0161i\u0107","raw_affiliation_strings":["AIT Austrian Institute of Technology GmbH,Vienna,Austria","AIT Austrian Institute of Technology GmbH, Vienna, Austria"],"affiliations":[{"raw_affiliation_string":"AIT Austrian Institute of Technology GmbH,Vienna,Austria","institution_ids":["https://openalex.org/I132118926"]},{"raw_affiliation_string":"AIT Austrian Institute of Technology GmbH, Vienna, Austria","institution_ids":["https://openalex.org/I132118926"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057223922","display_name":"Edgar A. Aguilar","orcid":"https://orcid.org/0000-0002-1177-9246"},"institutions":[{"id":"https://openalex.org/I132118926","display_name":"Austrian Institute of Technology","ror":"https://ror.org/04knbh022","country_code":"AT","type":"facility","lineage":["https://openalex.org/I132118926"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Edgar A. Aguilar","raw_affiliation_strings":["AIT Austrian Institute of Technology GmbH,Vienna,Austria","AIT Austrian Institute of Technology GmbH, Vienna, Austria"],"affiliations":[{"raw_affiliation_string":"AIT Austrian Institute of Technology GmbH,Vienna,Austria","institution_ids":["https://openalex.org/I132118926"]},{"raw_affiliation_string":"AIT Austrian Institute of Technology GmbH, Vienna, Austria","institution_ids":["https://openalex.org/I132118926"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052399473","display_name":"Dejan Ni\u010dkovi\u0107","orcid":"https://orcid.org/0000-0001-5468-0396"},"institutions":[{"id":"https://openalex.org/I132118926","display_name":"Austrian Institute of Technology","ror":"https://ror.org/04knbh022","country_code":"AT","type":"facility","lineage":["https://openalex.org/I132118926"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Dejan Ni\u010dkovi\u0107","raw_affiliation_strings":["AIT Austrian Institute of Technology GmbH,Vienna,Austria","AIT Austrian Institute of Technology GmbH, Vienna, Austria"],"affiliations":[{"raw_affiliation_string":"AIT Austrian Institute of Technology GmbH,Vienna,Austria","institution_ids":["https://openalex.org/I132118926"]},{"raw_affiliation_string":"AIT Austrian Institute of Technology GmbH, Vienna, Austria","institution_ids":["https://openalex.org/I132118926"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057473400","display_name":"Jyotirmoy V. Deshmukh","orcid":"https://orcid.org/0000-0003-4683-5540"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jyotirmoy V. Deshmukh","raw_affiliation_strings":["University of Southern California,Los Angeles,California,USA","University of Southern California, Los Angeles, California, USA"],"affiliations":[{"raw_affiliation_string":"University of Southern California,Los Angeles,California,USA","institution_ids":["https://openalex.org/I1174212"]},{"raw_affiliation_string":"University of Southern California, Los Angeles, California, USA","institution_ids":["https://openalex.org/I1174212"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5044356220"],"corresponding_institution_ids":["https://openalex.org/I1174212"],"apc_list":null,"apc_paid":null,"fwci":0.6279,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.72310312,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"6834","last_page":"6840"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10142","display_name":"Formal Methods in Verification","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10142","display_name":"Formal Methods in Verification","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9847000241279602,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.847555935382843},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7842036485671997},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.7095211744308472},{"id":"https://openalex.org/keywords/learning-automata","display_name":"Learning automata","score":0.5885540843009949},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5730816721916199},{"id":"https://openalex.org/keywords/automaton","display_name":"Automaton","score":0.5587403774261475},{"id":"https://openalex.org/keywords/rotation-formalisms-in-three-dimensions","display_name":"Rotation formalisms in three dimensions","score":0.5266218781471252},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.48768913745880127},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.4634195864200592},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.46157702803611755},{"id":"https://openalex.org/keywords/finite-state-machine","display_name":"Finite-state machine","score":0.45796018838882446},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.4312150478363037},{"id":"https://openalex.org/keywords/temporal-difference-learning","display_name":"Temporal difference learning","score":0.42594006657600403},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.41448381543159485},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.34964117407798767},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.24094653129577637},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.13206341862678528}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.847555935382843},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7842036485671997},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.7095211744308472},{"id":"https://openalex.org/C2776807809","wikidata":"https://www.wikidata.org/wiki/Q6510160","display_name":"Learning automata","level":3,"score":0.5885540843009949},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5730816721916199},{"id":"https://openalex.org/C112505250","wikidata":"https://www.wikidata.org/wiki/Q787116","display_name":"Automaton","level":2,"score":0.5587403774261475},{"id":"https://openalex.org/C171018156","wikidata":"https://www.wikidata.org/wiki/Q7370306","display_name":"Rotation formalisms in three dimensions","level":2,"score":0.5266218781471252},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.48768913745880127},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.4634195864200592},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46157702803611755},{"id":"https://openalex.org/C167822520","wikidata":"https://www.wikidata.org/wiki/Q176452","display_name":"Finite-state machine","level":2,"score":0.45796018838882446},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.4312150478363037},{"id":"https://openalex.org/C196340769","wikidata":"https://www.wikidata.org/wiki/Q7698910","display_name":"Temporal difference learning","level":3,"score":0.42594006657600403},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.41448381543159485},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.34964117407798767},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.24094653129577637},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.13206341862678528},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/cdc49753.2023.10383559","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cdc49753.2023.10383559","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 62nd IEEE Conference on Decision and Control (CDC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.800000011920929,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[{"id":"https://openalex.org/G2188898100","display_name":null,"funder_award_id":"SHF-2048094,CNS-1932620,CNS-2039087,FMitF-1837131,CCF-SHF-1932620","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8389257848","display_name":null,"funder_award_id":"H2020/2014-2020","funder_id":"https://openalex.org/F4320332999","funder_display_name":"Horizon 2020 Framework Programme"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332999","display_name":"Horizon 2020 Framework Programme","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W1533557569","https://openalex.org/W1941445455","https://openalex.org/W1967294943","https://openalex.org/W2049399166","https://openalex.org/W2078151802","https://openalex.org/W2091565802","https://openalex.org/W2151958719","https://openalex.org/W2462906003","https://openalex.org/W2524638160","https://openalex.org/W2567705466","https://openalex.org/W2620974420","https://openalex.org/W2676873924","https://openalex.org/W2798588334","https://openalex.org/W2803793821","https://openalex.org/W2895196950","https://openalex.org/W2913668833","https://openalex.org/W2918383326","https://openalex.org/W2963190533","https://openalex.org/W2963778636","https://openalex.org/W2964247868","https://openalex.org/W2966537673","https://openalex.org/W2990138404","https://openalex.org/W2995074243","https://openalex.org/W3004091789","https://openalex.org/W3026873144","https://openalex.org/W3038180127","https://openalex.org/W3080398910","https://openalex.org/W3092156990","https://openalex.org/W3120924814","https://openalex.org/W3213843225","https://openalex.org/W4287113179","https://openalex.org/W6638088447","https://openalex.org/W6677916085","https://openalex.org/W6751535212","https://openalex.org/W6752298494","https://openalex.org/W6766842884","https://openalex.org/W6797705063"],"related_works":["https://openalex.org/W1524177496","https://openalex.org/W2145363145","https://openalex.org/W2341346307","https://openalex.org/W1545451257","https://openalex.org/W2383312578","https://openalex.org/W4308702637","https://openalex.org/W1574991376","https://openalex.org/W3200605392","https://openalex.org/W61119710","https://openalex.org/W2808418668"],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1,37],"(RL)":[2],"is":[3,56,123],"a":[4,100,120,124,160],"popular":[5,142],"paradigm":[6],"for":[7,103,144,238],"synthesizing":[8],"controllers":[9],"in":[10,54,113,230],"environments":[11],"modeled":[12],"as":[13,65,149],"Markov":[14],"Decision":[15],"Processes":[16],"(MDPs).":[17],"The":[18],"RL":[19,55,162,207],"formulation":[20],"assumes":[21],"that":[22,27,44,57,59,159,178,180,209,223,243],"users":[23],"define":[24],"local":[25,86,106],"rewards":[26,47,62,87,107,188],"depend":[28],"only":[29],"on":[30],"the":[31,68,96,114,136,171,191,195,233,241,244,248],"current":[32],"state":[33,126],"(and":[34],"action),":[35],"and":[36,82,91,217,240],"algorithms":[38],"seek":[39],"to":[40,211],"find":[41],"control":[42,70,74,198,219],"policies":[43,58,213],"maximize":[45,60],"cumulative":[46,61],"along":[48],"system":[49,80,172],"trajectories.":[50],"An":[51],"implicit":[52],"assumption":[53],"are":[63,76,130],"desirable":[64],"they":[66],"meet":[67],"intended":[69],"objectives.":[71,250],"However,":[72],"most":[73],"objectives":[75,111],"global":[77],"properties":[78],"of":[79,94,116,152,170,193,232,235],"trajectories,":[81],"meeting":[83],"them":[84],"with":[85,132,205],"requires":[88],"tedious,":[89],"manual":[90],"error-prone":[92],"process":[93],"hand-crafting":[95],"rewards.":[97],"We":[98,157,174,200,221],"propose":[99],"new":[101],"algorithm":[102],"automatically":[104],"inferring":[105],"from":[108],"high-level":[109],"task":[110,146],"expressed":[112],"form":[115],"symbolic":[117,121,133],"automata":[118],"(SA);":[119],"automaton":[122],"finite":[125],"machine":[127],"where":[128],"edges":[129],"labeled":[131],"predicates":[134],"over":[135],"MDP":[137],"states.":[138],"SA":[139],"subsume":[140],"many":[141],"formalisms":[143],"expressing":[145],"objectives,":[147],"such":[148],"discrete-time":[150],"versions":[151],"Signal":[153],"Temporal":[154],"Logic":[155],"(STL).":[156],"assume":[158,166],"model-free":[161],"setting,":[163],"i.e.,":[164],"we":[165],"no":[167],"prior":[168],"knowledge":[169],"dynamics.":[173],"give":[175],"theoretical":[176],"results":[177],"establish":[179],"an":[181],"optimal":[182],"policy":[183,246],"learned":[184,245],"using":[185],"our":[186,203,224],"shaped":[187],"also":[189],"maximizes":[190],"probability":[192,242],"satisfying":[194],"given":[196],"SA-based":[197,249],"objective.":[199,220],"empirically":[201],"compare":[202],"approach":[204,225],"other":[206],"methods":[208,228],"try":[210],"learn":[212],"enforcing":[214],"temporal":[215],"logic":[216],"automata-based":[218],"demonstrate":[222],"outperforms":[226],"these":[227],"both":[229],"terms":[231],"number":[234],"iterations":[236],"required":[237],"convergence":[239],"satisfies":[247]},"counts_by_year":[{"year":2024,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2024-01-20T00:00:00"}