{"id":"https://openalex.org/W2915060045","doi":null,"title":"Knowledge and ignorance in reinforcement learning","display_name":"Knowledge and ignorance in reinforcement learning","publication_year":2011,"publication_date":"2011-01-01","ids":{"openalex":"https://openalex.org/W2915060045","mag":"2915060045"},"language":"en","primary_location":{"id":"mag:2915060045","doi":null,"is_oa":false,"landing_page_url":"https://www.ideals.illinois.edu/handle/2142/24337","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"volume":null,"type":"article","indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023802054","display_name":"Dan Roth","orcid":"https://orcid.org/0009-0002-1447-5173","observed_orcids":["https://orcid.org/0009-0002-1447-5173"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dan Roth","raw_affiliation_strings":["***University of Illinois at Urbana-Champaign"],"raw_orcid":"https://orcid.org/0009-0002-1447-5173","affiliations":[{"raw_affiliation_string":"***University of Illinois at Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111238931","display_name":"Deepak Ramachandran","orcid":"https://orcid.org/0000-0001-5412-6133","observed_orcids":["https://orcid.org/0000-0001-5412-6133"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Deepak Ramachandran","raw_affiliation_strings":["***University of Illinois at Urbana-Champaign"],"raw_orcid":"https://orcid.org/0000-0001-5412-6133","affiliations":[{"raw_affiliation_string":"***University of Illinois at Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":0.2947,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.72336054,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"121","last_page":"121"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inverse-reinforcement-learning","display_name":"inverse reinforcement learning","score":0.7799999713897705},{"id":"https://openalex.org/keywords/imitation-learning","display_name":"imitation learning","score":0.8989999890327454},{"id":"https://openalex.org/keywords/reward-shaping","display_name":"reward shaping","score":0.8539999723434448},{"id":"https://openalex.org/keywords/observation-model","display_name":"observation model","score":0.7919999957084656},{"id":"https://openalex.org/keywords/partial-observability","display_name":"partial observability","score":0.8650000095367432},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.6800000071525574},{"id":"https://openalex.org/keywords/reward-function-learning","display_name":"reward function learning","score":0.48399999737739563},{"id":"https://openalex.org/keywords/preference-elicitation","display_name":"preference elicitation","score":0.8799999952316284},{"id":"https://openalex.org/keywords/robotic-manipulation","display_name":"robotic manipulation","score":0.6050000190734863},{"id":"https://openalex.org/keywords/video-games","display_name":"video games","score":0.8090000152587891},{"id":"https://openalex.org/keywords/uncertainty-in-learning","display_name":"uncertainty in learning","score":0.3610000014305115},{"id":"https://openalex.org/keywords/bayesian-neural-networks","display_name":"Bayesian neural networks","score":0.34200000762939453},{"id":"https://openalex.org/keywords/deep-reinforcement-learning","display_name":"deep reinforcement learning","score":0.4399999976158142},{"id":"https://openalex.org/keywords/deep-q-network","display_name":"deep Q-network","score":0.3230000138282776},{"id":"https://openalex.org/keywords/ddpg","display_name":"deep deterministic policy gradient","score":0.41999998688697815}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8133031129837036},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6302427649497986},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.6096255779266357},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5318925380706787},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4911806285381317},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4481083154678345},{"id":"https://openalex.org/C72434380","wikidata":"https://www.wikidata.org/wiki/Q230930","display_name":"State space","level":2,"score":0.43983781337738037},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.2750329375267029},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11812135577201843},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:2915060045","doi":null,"is_oa":false,"landing_page_url":"https://www.ideals.illinois.edu/handle/2142/24337","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"display_name":"Peace, justice, and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.800000011920929}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3215946589","https://openalex.org/W2069855531","https://openalex.org/W7099390080","https://openalex.org/W3027397596","https://openalex.org/W1972336342","https://openalex.org/W2062525454","https://openalex.org/W7162349111","https://openalex.org/W7161983968","https://openalex.org/W2809461852","https://openalex.org/W7221148505"],"abstract_inverted_index":{"The":[0,171],"field":[1],"of":[2,70,74,166,197,249,297,365,376,391,402,438,518],"Reinforcement":[3,23,79,135,152,207],"Learning":[4,24,80,136,153,208],"is":[5,114,158,174,196,237,266,368,384,417,492],"concerned":[6],"with":[7,140,469],"teaching":[8],"agents":[9],"to":[10,14,118,133,138,159,175,186,210,218,262,269,283,313,326,332,371,387,424,552],"take":[11],"optimal":[12,184,340],"decisions":[13],"maximize":[15],"their":[16],"total":[17],"utility":[18],"in":[19,40,149,253,354,455,498,527,556],"complicated":[20,467],"environments.":[21],"A":[22],"problem,":[25],"generally":[26],"described":[27],"by":[28,224,243,462],"the":[29,48,51,55,68,78,85,89,150,156,161,180,183,193,220,230,247,250,276,302,334,339,363,430,439,485,495,515,525,528,533],"Markov":[30],"Decision":[31],"Process":[32],"formalism,":[33],"has":[34,93,145,279,348,408],"several":[35],"complex":[36],"interacting":[37],"components,":[38],"unlike":[39],"other":[41,240],"machine":[42],"learning":[43,100,178,335,407,482],"settings.":[44],"I":[45,61,82,130,202,378,452,473],"distinguish":[46],"three:":[47],"state-space/transition":[49],"model,":[50,91],"reward":[52,86,162,194,231,251,322,330,392,414],"function,":[53],"and":[54,88,223,255,272,290,351,394,433,444,512,531,544],"observation":[56,90,403,486],"model.":[57],"In":[58,127,293,321,449],"this":[59,128,211,311,450],"thesis,":[60,129],"present":[62,475],"a":[63,204,215,294,315,328,385,388,435,519,547],"framework":[64],"for":[65,99,287,397,481],"studying":[66],"how":[67,132,420,454],"state":[69,503],"knowledge":[71],"or":[72,125,464],"uncertainty":[73,526],"each":[75],"component":[76],"affects":[77],"process.":[81],"focus":[83],"on":[84,246,338,405,494],"function":[87,163,195,232,252,331],"which":[92,383],"traditionally":[94],"received":[95],"little":[96],"attention.":[97],"Algorithms":[98],"good":[101],"policies":[102,483],"when":[103,120,484],"these":[104,141,538],"components":[105],"are":[106,109,122,488],"completely":[107],"specified":[108],"well":[110],"understood.":[111],"However,":[112,191],"it":[113,226],"less":[115,412],"clear":[116],"what":[117],"do":[119,314],"they":[121],"unknown,":[123],"uncertain":[124],"irrelevant.":[126],"describe":[131,203,219,453],"adapt":[134],"algorithms":[137,396],"cope":[139],"situations.":[142],"\r\nRecently":[143],"there":[144],"been":[146,280,349,409],"great":[147],"interest":[148,199],"Inverse":[151,206],"problem":[154,286],"where":[155,179],"objective":[157],"learn":[160],"from":[164,188,233,239,275],"evidence":[165],"an":[167,189,259,425,476,557],"agent's":[168],"reward-maximizing":[169],"policy.":[170],"usual":[172],"goal":[173],"perform":[176,187],"apprenticeship":[177,263],"agent":[181,336],"learns":[182],"action":[185,234],"expert.":[190,277],"sometimes":[192],"independent":[198],"as":[200,258,342,344],"well.":[201],"Bayesian":[205,380],"approach":[209],"problem.":[212],"BIRL":[213,265,300],"uses":[214],"generative":[216],"model":[217,312,427,530],"decision-making":[221],"process":[222],"inverting":[225],"we":[227,324],"can":[228,459,505,523],"infer":[229],"observations.":[235],"It":[236,278],"distinguished":[238],"IRL":[241,298,306],"approaches":[242,539],"placing":[244],"emphasis":[245],"accuracy":[248,443],"itself,":[254],"not":[256],"just":[257],"intermediate":[260],"step":[261,517],"learning.":[264],"also":[267,309,474],"able":[268],"handle":[270],"incomplete":[271],"contradictory":[273],"information":[274],"applied":[281],"successfully":[282],"preference":[284],"elicitation":[285],"computer":[288],"games":[289],"robot":[291,549],"manipulation.":[292],"recent":[295],"comparison":[296],"approaches,":[299],"was":[301,550],"best":[303],"performing":[304],"general":[305],"algorithm.":[307],"\r\nI":[308],"extend":[310],"related":[316],"task,":[317],"Reward":[318,346,381,542],"Shaping":[319,543],".":[320],"shaping,":[323,393],"seek":[325],"adjust":[327],"known":[329],"make":[333],"converge":[337],"policy":[341],"fast":[343],"possible.":[345],"shaping":[347],"proposed":[350],"studied":[352,410],"previously":[353],"many":[355,373,499],"applications,":[356],"typically":[357],"using":[358],"additive":[359],"potential":[360],"functions.":[361,415],"However":[362],"requirement":[364],"absolute":[366],"policy-invariance":[367],"too":[369],"strict":[370],"admit":[372],"useful":[374],"cases":[375,457],"shaping.":[377],"define":[379],"Shaping,":[382],"generalization":[386],"soft":[389],"form":[390],"provide":[395],"achieving":[398],"it.":[399],"\r\nThe":[400],"impact":[401],"models":[404,468,487],"reinforcement":[406],"even":[411],"than":[413],"This":[416],"surpising,":[418],"considering":[419],"adding":[421],"partial":[422],"observability":[423],"MDP":[426,535],"blows":[428],"up":[429],"computational":[431],"complexity":[432],"hence":[434],"better":[436,502],"understanding":[437],"tradeoffs":[440],"between":[441],"representational":[442],"efficiency":[445],"would":[446],"be":[447,460,506],"helpful.":[448],"work,":[451],"certain":[456],"POMDPs":[458,501],"approximated":[461],"MDPs":[463],"slightly":[465],"more":[466],"bounded":[470],"performance":[471],"loss.":[472],"algorithm,":[477],"called":[478],"Smoothed":[479,490,545],"Q-Learning":[480],"uncertain.":[489],"Sarsa":[491],"based":[493],"idea":[496],"that":[497],"real-world":[500],"estimates":[504],"made":[507],"at":[508],"later":[509],"time":[510],"steps":[511],"thus":[513],"delaying":[514],"backup":[516],"temporal":[520],"difference-based":[521],"algorithm":[522],"shortcut":[524],"obervation":[529],"approximate":[532],"underlying":[534],"better.":[536],"\r\nCombining":[537],"together":[540],"(Bayesian":[541],"Sarsa),":[546],"mobile":[548],"trained":[551],"execute":[553],"delivery":[554],"tasks":[555],"office":[558],"environment.":[559]},"counts_by_year":[{"year":2012,"cited_by_count":1}],"updated_date":"2025-11-06T04:12:42.849631","created_date":"2019-02-21T00:00:00"}