{"id":"https://openalex.org/W2592496730","doi":"https://doi.org/10.1109/tsmc.2017.2671848","title":"Model Learning for Multistep Backward Prediction in Dyna-${Q}$ Learning","display_name":"Model Learning for Multistep Backward Prediction in Dyna-${Q}$ Learning","publication_year":2017,"publication_date":"2017-03-07","ids":{"openalex":"https://openalex.org/W2592496730","doi":"https://doi.org/10.1109/tsmc.2017.2671848","mag":"2592496730"},"language":"en","primary_location":{"id":"doi:10.1109/tsmc.2017.2671848","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tsmc.2017.2671848","pdf_url":null,"source":{"id":"https://openalex.org/S4210209078","display_name":"IEEE Transactions on Systems Man and Cybernetics Systems","issn_l":"2168-2216","issn":["2168-2216","2168-2232"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Systems, Man, and Cybernetics: Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061189209","display_name":"Kao\u2010Shing Hwang","orcid":"https://orcid.org/0000-0001-9234-4836"},"institutions":[{"id":"https://openalex.org/I142974352","display_name":"National Sun Yat-sen University","ror":"https://ror.org/00mjawt10","country_code":"TW","type":"education","lineage":["https://openalex.org/I142974352"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Kao-Shing Hwang","raw_affiliation_strings":["Department of Electrical Engineering, National Sun Yat-sen University, Kaohsiung, Taiwan"],"raw_orcid":"https://orcid.org/0000-0001-9234-4836","affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering, National Sun Yat-sen University, Kaohsiung, Taiwan","institution_ids":["https://openalex.org/I142974352"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001081733","display_name":"Wei\u2010Cheng Jiang","orcid":"https://orcid.org/0000-0003-4432-8801"},"institutions":[{"id":"https://openalex.org/I152743029","display_name":"Kaohsiung Medical University","ror":"https://ror.org/03gk81f96","country_code":"TW","type":"education","lineage":["https://openalex.org/I152743029"]},{"id":"https://openalex.org/I4210121532","display_name":"Ministry of Health and Welfare","ror":"https://ror.org/024w0ge69","country_code":"TW","type":"government","lineage":["https://openalex.org/I4210121532","https://openalex.org/I4405257903"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Wei-Cheng Jiang","raw_affiliation_strings":["Department of Healthcare Administration and Medical Informatic, Kaohsiung Medical University, Kaohisung, Taiwan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Healthcare Administration and Medical Informatic, Kaohsiung Medical University, Kaohisung, Taiwan","institution_ids":["https://openalex.org/I152743029","https://openalex.org/I4210121532"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100731680","display_name":"Yu-Jen Chen","orcid":"https://orcid.org/0000-0003-0288-0796"},"institutions":[{"id":"https://openalex.org/I148099254","display_name":"National Chung Cheng University","ror":"https://ror.org/0028v3876","country_code":"TW","type":"education","lineage":["https://openalex.org/I148099254"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Yu-Jen Chen","raw_affiliation_strings":["Department of Electrical Engineering, National Chung Cheng University, Chiayi, Taiwan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering, National Chung Cheng University, Chiayi, Taiwan","institution_ids":["https://openalex.org/I148099254"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082113600","display_name":"Iris Hwang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Iris Hwang","raw_affiliation_strings":["Innova Solutions, Taipei, Taiwan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Innova Solutions, Taipei, Taiwan","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2645,"currency":"USD","value_usd":2645},"apc_paid":null,"fwci":0.2281,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.65952507,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":"48","issue":"9","first_page":"1470","last_page":"1481"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9850999712944031,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9793999791145325,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.537105143070221},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3901370167732239}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.537105143070221},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3901370167732239}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tsmc.2017.2671848","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tsmc.2017.2671848","pdf_url":null,"source":{"id":"https://openalex.org/S4210209078","display_name":"IEEE Transactions on Systems Man and Cybernetics Systems","issn_l":"2168-2216","issn":["2168-2216","2168-2232"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Systems, Man, and Cybernetics: Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.49000000953674316,"display_name":"Life in Land","id":"https://metadata.un.org/sdg/15"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W1040999251","https://openalex.org/W1491843047","https://openalex.org/W1557517019","https://openalex.org/W1783866091","https://openalex.org/W1893871610","https://openalex.org/W1975987482","https://openalex.org/W1987725948","https://openalex.org/W1988707322","https://openalex.org/W1989962452","https://openalex.org/W1997733123","https://openalex.org/W1997880753","https://openalex.org/W2048226872","https://openalex.org/W2078196735","https://openalex.org/W2096001037","https://openalex.org/W2097082695","https://openalex.org/W2107726111","https://openalex.org/W2112483970","https://openalex.org/W2121863487","https://openalex.org/W2124674171","https://openalex.org/W2151250975","https://openalex.org/W2155044741","https://openalex.org/W2166265228","https://openalex.org/W2258059573","https://openalex.org/W3011120880","https://openalex.org/W4214717370","https://openalex.org/W4240328096","https://openalex.org/W4245108548","https://openalex.org/W6638076483","https://openalex.org/W6678672301","https://openalex.org/W6684100858","https://openalex.org/W6691886369","https://openalex.org/W6775686901"],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2093578348","https://openalex.org/W2376932109","https://openalex.org/W2382290278","https://openalex.org/W2049775471","https://openalex.org/W2350741829","https://openalex.org/W3004735627"],"abstract_inverted_index":{"A":[0],"model-based":[1],"reinforcement":[2],"learning":[3,11,83,106,251,268],"(RL)":[4],"method":[5,84,99,107,122,278,296],"which":[6,151],"interplays":[7],"direct":[8],"and":[9,33,135,226,274],"indirect":[10,130],"to":[12,29,44,46,56,65,71,102,111,212,239,261],"update${Q}$functions":[13],"is":[14,18,42,89,100,231,237,253,279,297],"proposed.":[15],"The":[16,272,286],"environment":[17],"approximated":[19],"by":[20,141],"a":[21,80,178,183,223],"virtual":[22,40,118],"model":[23,41,82,92,105],"that":[24,160,181,233,290],"can":[25,269],"predict":[26],"the":[27,30,34,37,75,93,104,109,113,117,120,127,145,147,154,164,202,206,235,244,250,266,291],"transition":[28,94,114],"next":[31],"state":[32,166,185],"reward":[35],"of":[36,69,74,129,150,246,276,294],"domain.":[38],"This":[39,255],"used":[43,55],"train${Q}$functions":[45],"accelerate":[47],"policy":[48,267],"learning.":[49,131],"Lookup":[50],"table":[51],"methods":[52,63,260],"are":[53,137,158,209,220,227],"usually":[54],"establish":[57],"such":[58,191],"environmental":[59],"models,":[60],"but":[61],"these":[62,247],"need":[64],"collect":[66],"tremendous":[67],"amounts":[68],"experiences":[70],"enumerate":[72],"responses":[73],"environment.":[76],"In":[77],"this":[78,263],"paper,":[79],"stochastic":[81],"based":[85,221],"on":[86,222],"tree":[87],"structures":[88],"presented.":[90],"To":[91],"probability,":[95],"an":[96,188,193],"online":[97],"clustering":[98],"applied":[101],"equip":[103],"with":[108],"abilities":[110],"evaluate":[112],"probability.":[115],"By":[116],"model,":[119],"RL":[121],"produces":[123],"simulated":[124,133,224,241],"experience":[125,225,242],"in":[126,199,282],"stage":[128,208],"Since":[132],"transitions":[134],"backups":[136,157],"more":[138],"usefully":[139],"focused":[140],"working":[142],"backward":[143],"from":[144],"state-action,":[146],"pair":[148],"estimated${Q}$value":[149],"changes":[152,203],"significantly,":[153],"useful":[155],"one-step":[156],"actions":[159],"lead":[161],"directly":[162],"into":[163],"one":[165],"whose":[167],"value":[168],"has":[169],"already":[170],"obviously":[171,298],"been":[172],"changed.":[173],"This,":[174],"however,":[175],"may":[176,186],"induce":[177],"false":[179],"positive;":[180],"is,":[182],"backup":[184],"be":[187],"invalid":[189],"state,":[190,197],"as":[192],"absorbing":[194,248],"or":[195],"terminal":[196],"especially":[198],"cases":[200],"where":[201],"of${Q}$values":[204],"at":[205],"planning":[207],"still":[210],"needed":[211],"put":[213],"back":[214],"for":[215],"ranking":[216],"even":[217],"though":[218],"they":[219],"possibly":[228],"erroneous.":[229],"It":[230],"obvious":[232],"when":[234],"agent":[236],"attracted":[238],"generate":[240],"around":[243],"area":[245],"states,":[249],"efficiency":[252],"deteriorated.":[254],"paper":[256],"proposes":[257],"three":[258,283],"detecting":[259],"solve":[262],"problem.":[264],"Moreover,":[265],"speed":[270],"up.":[271],"effectiveness":[273],"generality":[275],"our":[277,295],"further":[280],"demonstrated":[281],"numerical":[284],"simulations.":[285],"simulation":[287],"results":[288],"demonstrate":[289],"training":[292],"rate":[293],"improved.":[299]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":3},{"year":2019,"cited_by_count":1}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}