{"id":"https://openalex.org/W4385240323","doi":"https://doi.org/10.1177/02783649231185165","title":"Stabilizing deep Q-learning with Q-graph-based bounds","display_name":"Stabilizing deep Q-learning with Q-graph-based bounds","publication_year":2023,"publication_date":"2023-07-25","ids":{"openalex":"https://openalex.org/W4385240323","doi":"https://doi.org/10.1177/02783649231185165"},"language":"en","primary_location":{"id":"doi:10.1177/02783649231185165","is_oa":false,"landing_page_url":"https://doi.org/10.1177/02783649231185165","pdf_url":null,"source":{"id":"https://openalex.org/S73484101","display_name":"The International Journal of Robotics Research","issn_l":"0278-3649","issn":["0278-3649","1741-3176"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The International Journal of Robotics Research","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091621505","display_name":"Sabrina Hoppe","orcid":"https://orcid.org/0000-0001-6958-8015"},"institutions":[{"id":"https://openalex.org/I889804353","display_name":"Robert Bosch (Germany)","ror":"https://ror.org/01fe0jt45","country_code":"DE","type":"company","lineage":["https://openalex.org/I889804353"]},{"id":"https://openalex.org/I100066346","display_name":"University of Stuttgart","ror":"https://ror.org/04vnq7t77","country_code":"DE","type":"education","lineage":["https://openalex.org/I100066346"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Sabrina Hoppe","raw_affiliation_strings":["Bosch Center for Artificial Intelligence, Robert Bosch GmbH, Renningen, Germany","Machine Learning and Robotics Lab, University of Stuttgart, Stuttgart, Germany"],"affiliations":[{"raw_affiliation_string":"Bosch Center for Artificial Intelligence, Robert Bosch GmbH, Renningen, Germany","institution_ids":["https://openalex.org/I889804353"]},{"raw_affiliation_string":"Machine Learning and Robotics Lab, University of Stuttgart, Stuttgart, Germany","institution_ids":["https://openalex.org/I100066346"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086334164","display_name":"Markus Giftthaler","orcid":"https://orcid.org/0000-0002-1839-9449"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Markus Giftthaler","raw_affiliation_strings":["Google Germany GmbH, Munich, Germany"],"affiliations":[{"raw_affiliation_string":"Google Germany GmbH, Munich, Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035020090","display_name":"Robert Krug","orcid":null},"institutions":[{"id":"https://openalex.org/I889804353","display_name":"Robert Bosch (Germany)","ror":"https://ror.org/01fe0jt45","country_code":"DE","type":"company","lineage":["https://openalex.org/I889804353"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Robert Krug","raw_affiliation_strings":["Bosch Center for Artificial Intelligence, Robert Bosch GmbH, Renningen, Germany"],"affiliations":[{"raw_affiliation_string":"Bosch Center for Artificial Intelligence, Robert Bosch GmbH, Renningen, Germany","institution_ids":["https://openalex.org/I889804353"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5065672819","display_name":"Marc Toussaint","orcid":"https://orcid.org/0000-0002-5487-6767"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Marc Toussaint","raw_affiliation_strings":["Learning and Intelligent Systems Group, TU Berlin, Berlin, Germany"],"affiliations":[{"raw_affiliation_string":"Learning and Intelligent Systems Group, TU Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I4577782"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5091621505"],"corresponding_institution_ids":["https://openalex.org/I100066346","https://openalex.org/I889804353"],"apc_list":null,"apc_paid":null,"fwci":0.1748,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.54177653,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":"42","issue":"9","first_page":"633","last_page":"654"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7628346085548401},{"id":"https://openalex.org/keywords/q-learning","display_name":"Q-learning","score":0.6586495637893677},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6403126120567322},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.5622356534004211},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.5434492230415344},{"id":"https://openalex.org/keywords/hyperparameter","display_name":"Hyperparameter","score":0.5176727771759033},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4886660873889923},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.4326810836791992},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.3649309277534485},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.328605592250824},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.3233329653739929},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.2937353849411011},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.21429985761642456}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7628346085548401},{"id":"https://openalex.org/C188116033","wikidata":"https://www.wikidata.org/wiki/Q2664563","display_name":"Q-learning","level":3,"score":0.6586495637893677},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6403126120567322},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.5622356534004211},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.5434492230415344},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.5176727771759033},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4886660873889923},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.4326810836791992},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3649309277534485},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.328605592250824},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3233329653739929},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.2937353849411011},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.21429985761642456},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1177/02783649231185165","is_oa":false,"landing_page_url":"https://doi.org/10.1177/02783649231185165","pdf_url":null,"source":{"id":"https://openalex.org/S73484101","display_name":"The International Journal of Robotics Research","issn_l":"0278-3649","issn":["0278-3649","1741-3176"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The International Journal of Robotics Research","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4300000071525574,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W15078749","https://openalex.org/W32403112","https://openalex.org/W41554520","https://openalex.org/W92261402","https://openalex.org/W1646707810","https://openalex.org/W1677182931","https://openalex.org/W1967948264","https://openalex.org/W1978549191","https://openalex.org/W2105360813","https://openalex.org/W2107726111","https://openalex.org/W2108598243","https://openalex.org/W2110697446","https://openalex.org/W2145339207","https://openalex.org/W2232305727","https://openalex.org/W2746553466","https://openalex.org/W2754517384","https://openalex.org/W2783824884","https://openalex.org/W2911114506","https://openalex.org/W2959488596","https://openalex.org/W2963859097","https://openalex.org/W2963940579","https://openalex.org/W2964333597","https://openalex.org/W2967717386","https://openalex.org/W2967727187","https://openalex.org/W2990460121","https://openalex.org/W2998069546","https://openalex.org/W2998461398","https://openalex.org/W2999905431","https://openalex.org/W3011120880","https://openalex.org/W3099530328","https://openalex.org/W3130717831","https://openalex.org/W3132045372","https://openalex.org/W3164005523"],"related_works":["https://openalex.org/W2101748387","https://openalex.org/W3096874164","https://openalex.org/W2357975469","https://openalex.org/W4295309597","https://openalex.org/W4386337009","https://openalex.org/W2937181779","https://openalex.org/W4210794429","https://openalex.org/W4287719180","https://openalex.org/W3004055318","https://openalex.org/W3105659172"],"abstract_inverted_index":{"State-of-the":[0],"art":[1],"deep":[2,26],"reinforcement":[3],"learning":[4,158],"has":[5,76],"enabled":[6],"autonomous":[7],"agents":[8],"to":[9,66,89,161,189],"learn":[10],"complex":[11],"strategies":[12],"from":[13,108],"scratch":[14],"on":[15,39,71,169,203],"many":[16],"problems":[17],"including":[18,175],"continuous":[19,172],"control":[20,173],"tasks.":[21],"Deep":[22],"Q-networks":[23],"(DQN)":[24],"and":[25,49,78,106,151,213],"deterministic":[27],"policy":[28],"gradients":[29],"(DDPGs)":[30],"are":[31,36,141],"two":[32],"such":[33,115],"algorithms":[34],"which":[35,132],"both":[37],"based":[38],"Q-learning.":[40],"They":[41],"therefore":[42],"all":[43],"share":[44],"function":[45],"approximation,":[46],"off-policy":[47],"behavior,":[48],"bootstrapping\u2014the":[50],"constituents":[51],"of":[52,83,92,104,200,217],"the":[53,72,81,90,109,117,133,145,148,198],"so-called":[54],"deadly":[55],"triad":[56],"that":[57,80,94,101,116],"is":[58,87],"known":[59],"for":[60,131,144],"its":[61],"convergence":[62],"issues.":[63],"We":[64,98,165],"suggest":[65],"take":[67],"a":[68,102,125,170,186,204],"graph":[69,86,111,120],"perspective":[70],"data":[73,85,110],"an":[74,209],"agent":[75],"collected":[77],"show":[79,166],"structure":[82],"this":[84],"linked":[88],"degree":[91],"divergence":[93],"can":[95,112,121,135,159],"be":[96,113,122,136],"expected.":[97],"further":[99,167],"demonstrate":[100,197],"subset":[103],"states":[105],"actions":[107],"selected":[114],"resulting":[118],"finite":[119],"interpreted":[123],"as":[124,183,185],"simplified":[126],"Markov":[127],"decision":[128],"process":[129],"(MDP)":[130],"Q-values":[134,140,146],"computed":[137],"analytically.":[138],"These":[139],"lower":[142],"bounds":[143,154],"in":[147,155],"original":[149],"problem,":[150],"enforcing":[152],"these":[153],"temporal":[156],"difference":[157],"help":[160],"prevent":[162],"soft":[163],"divergence.":[164],"effects":[168],"simulated":[171],"task,":[174],"improved":[176],"sample":[177],"efficiency,":[178],"increased":[179],"robustness":[180],"toward":[181],"hyperparameters":[182],"well":[184],"better":[187],"ability":[188],"cope":[190],"with":[191,208],"limited":[192],"replay":[193],"memory.":[194],"Finally,":[195],"we":[196],"benefits":[199],"our":[201],"method":[202],"large":[205],"robotic":[206],"benchmark":[207],"industrial":[210],"assembly":[211],"task":[212],"approximately":[214],"60":[215],"h":[216],"real-world":[218],"interaction.":[219]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}