{"id":"https://openalex.org/W7066096367","doi":null,"title":"Explaining the future context of deep reinforcement learning agents\u2019 decision-making","display_name":"Explaining the future context of deep reinforcement learning agents\u2019 decision-making","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W7066096367"},"language":"en","primary_location":{"id":"pmh:oai:eprints.soton.ac.uk:502074","doi":null,"is_oa":true,"landing_page_url":"https://eprints.soton.ac.uk/id/eprint/502074/1/archival_phd_thesis.pdf","pdf_url":"https://eprints.soton.ac.uk/502074/1/archival_phd_thesis.pdf","source":{"id":"https://openalex.org/S4306401019","display_name":"ePrints Soton (University of Southampton)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I43439940","host_organization_name":"University of Southampton","host_organization_lineage":["https://openalex.org/I43439940"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"},"volume":null,"type":"dissertation","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://eprints.soton.ac.uk/502074/1/archival_phd_thesis.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108585735","display_name":"Mark Towers","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Towers, Mark","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5108585735"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/deep-reinforcement-learning","display_name":"deep reinforcement learning","score":0.8500000238418579},{"id":"https://openalex.org/keywords/explainable-ai","display_name":"explainable AI","score":0.7910000085830688},{"id":"https://openalex.org/keywords/q-values","display_name":"Q-value","score":0.6470000147819519},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6629999876022339},{"id":"https://openalex.org/keywords/atari-games","display_name":"Atari games","score":0.6579999923706055},{"id":"https://openalex.org/keywords/agent-memory","display_name":"agent memory","score":0.39899998903274536}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8661999702453613},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6669999957084656},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6244000196456909},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5407999753952026},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5293999910354614},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5077000260353088},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.4309999942779541},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.39890000224113464},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.38440001010894775},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.350600004196167},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3377000093460083},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3176000118255615},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3174999952316284},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3100000023841858},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.3091000020503998},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.2964000105857849}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:eprints.soton.ac.uk:502074","doi":null,"is_oa":true,"landing_page_url":"https://eprints.soton.ac.uk/id/eprint/502074/1/archival_phd_thesis.pdf","pdf_url":"https://eprints.soton.ac.uk/502074/1/archival_phd_thesis.pdf","source":{"id":"https://openalex.org/S4306401019","display_name":"ePrints Soton (University of Southampton)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I43439940","host_organization_name":"University of Southampton","host_organization_lineage":["https://openalex.org/I43439940"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"}],"best_oa_location":{"id":"pmh:oai:eprints.soton.ac.uk:502074","doi":null,"is_oa":true,"landing_page_url":"https://eprints.soton.ac.uk/id/eprint/502074/1/archival_phd_thesis.pdf","pdf_url":"https://eprints.soton.ac.uk/502074/1/archival_phd_thesis.pdf","source":{"id":"https://openalex.org/S4306401019","display_name":"ePrints Soton (University of Southampton)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I43439940","host_organization_name":"University of Southampton","host_organization_lineage":["https://openalex.org/I43439940"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7066096367.pdf","grobid_xml":"https://content.openalex.org/works/W7066096367.grobid-xml"},"referenced_works_count":37,"referenced_works":["https://openalex.org/W2736601468","https://openalex.org/W2774522520","https://openalex.org/W2781726626","https://openalex.org/W2891830784","https://openalex.org/W2902098903","https://openalex.org/W2903808828","https://openalex.org/W2905342215","https://openalex.org/W2921136632","https://openalex.org/W2960876848","https://openalex.org/W3001279689","https://openalex.org/W3002769442","https://openalex.org/W3030163527","https://openalex.org/W3037207827","https://openalex.org/W3046949396","https://openalex.org/W3084646500","https://openalex.org/W3089162373","https://openalex.org/W3090586738","https://openalex.org/W3135560598","https://openalex.org/W4283220714","https://openalex.org/W4285483432","https://openalex.org/W4309183395","https://openalex.org/W4383649780","https://openalex.org/W4385768230","https://openalex.org/W4388923662","https://openalex.org/W4392196196","https://openalex.org/W4399553916","https://openalex.org/W4401023564","https://openalex.org/W4403479172","https://openalex.org/W4404330874","https://openalex.org/W4404341399","https://openalex.org/W4404368393","https://openalex.org/W4405095150","https://openalex.org/W4407091688","https://openalex.org/W6891880838","https://openalex.org/W6958662716","https://openalex.org/W7027579263","https://openalex.org/W7124276366"],"related_works":["https://openalex.org/W4402782681","https://openalex.org/W4280488708","https://openalex.org/W6894195468","https://openalex.org/W7056602424","https://openalex.org/W4403479172","https://openalex.org/W4406192858","https://openalex.org/W2914831223","https://openalex.org/W3092043450","https://openalex.org/W7202261782","https://openalex.org/W4393156810"],"abstract_inverted_index":{"Deep":[0],"reinforcement":[1,196,291],"learning":[2,197],"has":[3,61],"achieved":[4],"superhuman":[5],"performance":[6],"in":[7,39,115,163,310,356,366],"numerous":[8],"environments.":[9,117],"Despite":[10],"these":[11,101,129],"advances,":[12],"there":[13],"are":[14,149],"limited":[15,144],"tools":[16],"to":[17,32,52,70,112,166,182,184,239,254,261,305,318],"understand":[18],"why":[19],"agents":[20,31,198,267],"make":[21],"decisions.":[22],"A":[23],"central":[24],"issue":[25],"is":[26,50],"how":[27,69,159],"specific":[28],"actions":[29],"enable":[30],"collect":[33,172],"rewards":[34,209,271,309],"or":[35,178,212],"achieve":[36],"goals":[37],"far":[38],"the":[40,72,141,150,164,204,227,237,273,307,328,367],"future.":[41],"Understanding":[42],"this":[43],"future":[44,73,190,205,232,248,308],"context":[45,74],"for":[46,78,124,230,246,272,284,373],"an":[47,160,187,219,241,343,348,375],"agent's":[48,220,242,345,376],"decision-making":[49,77,245],"critical":[51],"explaining":[53],"their":[54,94,168,201,208,269],"choices.":[55],"To":[56],"date,":[57],"however,":[58],"little":[59],"research":[60],"explored":[62],"such":[63],"temporal":[64,321,361],"explanations.":[65],"Therefore,":[66],"we":[67,216,279,296,330],"investigate":[68],"explain":[71,167,240],"of":[75,85,128,137,158,290,350],"agents\u2019":[76],"both":[79,357],"pretrained":[80,125],"agents,":[81,91],"using":[82,301],"a":[83,134,156,256,287,298,332],"memory":[84,157],"past":[86,165,176],"behaviour,":[87],"and":[88,109,244],"architecturally":[89],"modified":[90],"explicitly":[92],"outputting":[93],"next":[95,274],"$N$":[96],"expected":[97,228,270],"rewards.":[98],"We":[99,118,171,192,313,353],"evaluate":[100,319],"explanations":[102,123,153,283,322,362],"with":[103,286],"user":[104,316,358],"surveys,":[105,359],"finding":[106],"them":[107,181],"preferred":[108,364],"more":[110,371],"effective":[111,372],"baseline":[113,325],"algorithms":[114],"Atari":[116],"develop":[119],"three":[120,281],"novel":[121,257,282,333],"video-based":[122],"agents.":[126],"Two":[127],"require":[130],"no":[131],"domain":[132,145],"knowledge,":[133],"common":[135],"feature":[136],"prior":[138],"work,":[139],"while":[140],"third":[142],"incorporates":[143],"knowledge.":[146],"These":[147],"approaches":[148],"first":[151],"local":[152],"that":[154,194,218,355],"use":[155],"agent":[161],"acted":[162],"current":[169],"decision-making.":[170,352],"similar":[173],"decisions":[174],"from":[175,347],"states":[177],"skills,":[179],"showcasing":[180],"users":[183,285,340],"help":[185],"visualise":[186],"action\u2019s":[188],"possible":[189],"outcomes.":[191],"identify":[193,342],"deep":[195],"implicitly":[199],"compute":[200],"beliefs":[202],"about":[203],"when":[206],"predicting":[207],"(i.e.,":[210],"Q-value":[211,221],"State-value).":[213],"From":[214,277],"this,":[215,278],"prove":[217],"can":[222],"be":[223],"transformed":[224],"into":[225],"computing":[226],"reward":[229],"each":[231],"timestep.":[233],"This":[234,250],"opens":[235],"up":[236],"opportunity":[238],"confidence":[243],"individual":[247],"timesteps.":[249,276],"innovation":[251],"allows":[252],"us":[253],"propose":[255,297,331],"training":[258],"algorithm":[259],"referred":[260],"as":[262],"Temporal":[263],"Reward":[264],"Decomposition,":[265],"where":[266,339],"output":[268],"N":[275],"pioneer":[280],"strong":[288],"understanding":[289],"learning.":[292],"For":[293],"non-technical":[294],"users,":[295],"fourth":[299],"explanation":[300,349],"Large":[302],"Language":[303],"Models":[304],"summarise":[306],"natural":[311],"language.":[312],"conduct":[314],"two":[315,324],"surveys":[317],"our":[320,360],"against":[323],"algorithms.":[326],"In":[327],"second,":[329,368],"evaluation":[334],"methodology":[335],"inspired":[336],"by":[337],"debugging,":[338],"must":[341],"unknown":[344],"goal":[346],"its":[351],"find":[354],"were":[363,369],"and,":[365],"significantly":[370],"determining":[374],"goal.":[377]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-10T00:00:00"}