{"id":"https://openalex.org/W1964534398","doi":"https://doi.org/10.1109/tcyb.2014.2313655","title":"Acceleration of Reinforcement Learning by Policy Evaluation Using Nonstationary Iterative Method","display_name":"Acceleration of Reinforcement Learning by Policy Evaluation Using Nonstationary Iterative Method","publication_year":2014,"publication_date":"2014-04-10","ids":{"openalex":"https://openalex.org/W1964534398","doi":"https://doi.org/10.1109/tcyb.2014.2313655","mag":"1964534398","pmid":"https://pubmed.ncbi.nlm.nih.gov/24733037"},"language":"en","primary_location":{"id":"doi:10.1109/tcyb.2014.2313655","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcyb.2014.2313655","pdf_url":null,"source":{"id":"https://openalex.org/S4210191041","display_name":"IEEE Transactions on Cybernetics","issn_l":"2168-2267","issn":["2168-2267","2168-2275"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cybernetics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://hdl.handle.net/2433/192769","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024169373","display_name":"Kei Senda","orcid":"https://orcid.org/0000-0001-5720-6155"},"institutions":[{"id":"https://openalex.org/I22299242","display_name":"Kyoto University","ror":"https://ror.org/02kpeqv85","country_code":"JP","type":"education","lineage":["https://openalex.org/I22299242"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Kei Senda","raw_affiliation_strings":["Department of Aeronautics and Astronautics, Kyoto University, Kyoto, Japan","Department of Aeronautics and Astronautics, Kyoto University, Kyoto, JAPAN"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Aeronautics and Astronautics, Kyoto University, Kyoto, Japan","institution_ids":["https://openalex.org/I22299242"]},{"raw_affiliation_string":"Department of Aeronautics and Astronautics, Kyoto University, Kyoto, JAPAN","institution_ids":["https://openalex.org/I22299242"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007419776","display_name":"Suguru Hattori","orcid":null},"institutions":[{"id":"https://openalex.org/I22299242","display_name":"Kyoto University","ror":"https://ror.org/02kpeqv85","country_code":"JP","type":"education","lineage":["https://openalex.org/I22299242"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Suguru Hattori","raw_affiliation_strings":["Department of Aeronautics and Astronautics, Kyoto University, Kyoto, Japan","Department of Aeronautics and Astronautics, Kyoto University, Kyoto, JAPAN"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Aeronautics and Astronautics, Kyoto University, Kyoto, Japan","institution_ids":["https://openalex.org/I22299242"]},{"raw_affiliation_string":"Department of Aeronautics and Astronautics, Kyoto University, Kyoto, JAPAN","institution_ids":["https://openalex.org/I22299242"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048198530","display_name":"Toru Hishinuma","orcid":"https://orcid.org/0000-0002-6922-1595"},"institutions":[{"id":"https://openalex.org/I22299242","display_name":"Kyoto University","ror":"https://ror.org/02kpeqv85","country_code":"JP","type":"education","lineage":["https://openalex.org/I22299242"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Toru Hishinuma","raw_affiliation_strings":["Department of Aeronautics and Astronautics, Kyoto University, Kyoto, Japan","Department of Aeronautics and Astronautics, Kyoto University, Kyoto, JAPAN"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Aeronautics and Astronautics, Kyoto University, Kyoto, Japan","institution_ids":["https://openalex.org/I22299242"]},{"raw_affiliation_string":"Department of Aeronautics and Astronautics, Kyoto University, Kyoto, JAPAN","institution_ids":["https://openalex.org/I22299242"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024603090","display_name":"Takehisa Kohda","orcid":null},"institutions":[{"id":"https://openalex.org/I22299242","display_name":"Kyoto University","ror":"https://ror.org/02kpeqv85","country_code":"JP","type":"education","lineage":["https://openalex.org/I22299242"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Takehisa Kohda","raw_affiliation_strings":["Department of Aeronautics and Astronautics, Kyoto University, Kyoto, Japan","Department of Aeronautics and Astronautics, Kyoto University, Kyoto, JAPAN"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Aeronautics and Astronautics, Kyoto University, Kyoto, Japan","institution_ids":["https://openalex.org/I22299242"]},{"raw_affiliation_string":"Department of Aeronautics and Astronautics, Kyoto University, Kyoto, JAPAN","institution_ids":["https://openalex.org/I22299242"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I22299242"],"apc_list":{"value":2645,"currency":"USD","value_usd":2645},"apc_paid":null,"fwci":2.1204,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.86394283,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"44","issue":"12","first_page":"2696","last_page":"2705"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11206","display_name":"Model Reduction and Neural Networks","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11522","display_name":"VLSI and FPGA Design Techniques","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7596102952957153},{"id":"https://openalex.org/keywords/subspace-topology","display_name":"Subspace topology","score":0.6660164594650269},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6479542255401611},{"id":"https://openalex.org/keywords/iterative-method","display_name":"Iterative method","score":0.6179193258285522},{"id":"https://openalex.org/keywords/acceleration","display_name":"Acceleration","score":0.587853193283081},{"id":"https://openalex.org/keywords/krylov-subspace","display_name":"Krylov subspace","score":0.5180795192718506},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.4641047716140747},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.4149687886238098},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2989680767059326},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.24396365880966187}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7596102952957153},{"id":"https://openalex.org/C32834561","wikidata":"https://www.wikidata.org/wiki/Q660730","display_name":"Subspace topology","level":2,"score":0.6660164594650269},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6479542255401611},{"id":"https://openalex.org/C159694833","wikidata":"https://www.wikidata.org/wiki/Q2321565","display_name":"Iterative method","level":2,"score":0.6179193258285522},{"id":"https://openalex.org/C117896860","wikidata":"https://www.wikidata.org/wiki/Q11376","display_name":"Acceleration","level":2,"score":0.587853193283081},{"id":"https://openalex.org/C147060835","wikidata":"https://www.wikidata.org/wiki/Q1757151","display_name":"Krylov subspace","level":3,"score":0.5180795192718506},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4641047716140747},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4149687886238098},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2989680767059326},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.24396365880966187},{"id":"https://openalex.org/C74650414","wikidata":"https://www.wikidata.org/wiki/Q11397","display_name":"Classical mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003198","descriptor_name":"Computer Simulation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003198","descriptor_name":"Computer Simulation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003198","descriptor_name":"Computer Simulation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003661","descriptor_name":"Decision Support Techniques","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003661","descriptor_name":"Decision Support Techniques","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003661","descriptor_name":"Decision Support Techniques","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D005246","descriptor_name":"Feedback","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005246","descriptor_name":"Feedback","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005246","descriptor_name":"Feedback","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008962","descriptor_name":"Models, Theoretical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D008962","descriptor_name":"Models, Theoretical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D008962","descriptor_name":"Models, Theoretical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D011340","descriptor_name":"Problem Solving","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D011340","descriptor_name":"Problem Solving","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D011340","descriptor_name":"Problem Solving","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012054","descriptor_name":"Reinforcement, Psychology","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012054","descriptor_name":"Reinforcement, Psychology","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012054","descriptor_name":"Reinforcement, Psychology","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":3,"locations":[{"id":"doi:10.1109/tcyb.2014.2313655","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcyb.2014.2313655","pdf_url":null,"source":{"id":"https://openalex.org/S4210191041","display_name":"IEEE Transactions on Cybernetics","issn_l":"2168-2267","issn":["2168-2267","2168-2275"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cybernetics","raw_type":"journal-article"},{"id":"pmid:24733037","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/24733037","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on cybernetics","raw_type":"Journal Article"},{"id":"pmh:oai:irdb.nii.ac.jp:01221:0000151813","is_oa":true,"landing_page_url":"http://hdl.handle.net/2433/192769","pdf_url":null,"source":{"id":"https://openalex.org/S7407056385","display_name":"Institutional Repositories DataBase (IRDB)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I184597095","host_organization_name":"National Institute of Informatics","host_organization_lineage":["https://openalex.org/I184597095"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"IEEE transactions on cybernetics","raw_type":"journal article"}],"best_oa_location":{"id":"pmh:oai:irdb.nii.ac.jp:01221:0000151813","is_oa":true,"landing_page_url":"http://hdl.handle.net/2433/192769","pdf_url":null,"source":{"id":"https://openalex.org/S7407056385","display_name":"Institutional Repositories DataBase (IRDB)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I184597095","host_organization_name":"National Institute of Informatics","host_organization_lineage":["https://openalex.org/I184597095"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"IEEE transactions on cybernetics","raw_type":"journal article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1479729523","display_name":"Understanding of Adaptive Capability in Insect Flight that Nonlinearly Interacts with Generated Flow-field by Using Robotic Biology","funder_award_id":"25289062","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"}],"funders":[{"id":"https://openalex.org/F4320334764","display_name":"Japan Society for the Promotion of Science","ror":"https://ror.org/00hhkn466"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W1506342804","https://openalex.org/W1571187512","https://openalex.org/W1576452626","https://openalex.org/W1601081659","https://openalex.org/W2073384958","https://openalex.org/W2118556122","https://openalex.org/W2121863487","https://openalex.org/W2132351269","https://openalex.org/W2143476164","https://openalex.org/W2155216327","https://openalex.org/W2161795906","https://openalex.org/W2315554190","https://openalex.org/W2316564661","https://openalex.org/W2400458653","https://openalex.org/W2487144912","https://openalex.org/W2586680856","https://openalex.org/W3027095131","https://openalex.org/W4206097460","https://openalex.org/W4214717370","https://openalex.org/W4285719527","https://openalex.org/W4299401133","https://openalex.org/W6683841269"],"related_works":["https://openalex.org/W1609431451","https://openalex.org/W2149246659","https://openalex.org/W2016713983","https://openalex.org/W1965616507","https://openalex.org/W2798351323","https://openalex.org/W4300721448","https://openalex.org/W2650947288","https://openalex.org/W2117065662","https://openalex.org/W2963317066","https://openalex.org/W1675901148"],"abstract_inverted_index":{"Typical":[0],"methods":[1],"for":[2,19],"solving":[3],"reinforcement":[4],"learning":[5,25],"problems":[6],"iterate":[7],"two":[8],"steps,":[9],"policy":[10,13,21],"evaluation":[11,22],"and":[12,94],"improvement.":[14],"This":[15,79],"paper":[16,80],"proposes":[17],"algorithms":[18,29,45,58,84],"the":[20,33,61],"to":[23,51],"improve":[24],"efficiency.":[26],"The":[27,44],"proposed":[28],"are":[30,49,69],"based":[31,46,59,66,85],"on":[32,47,60,67,86],"Krylov":[34],"Subspace":[35],"Method":[36],"(KSM),":[37],"which":[38],"is":[39],"a":[40],"nonstationary":[41],"iterative":[42,63],"method.":[43],"KSM":[48,68,87],"tens":[50],"hundreds":[52],"times":[53],"more":[54,71,89],"efficient":[55,72,90],"than":[56,73],"existing":[57],"stationary":[62],"methods.":[64],"Algorithms":[65],"far":[70],"they":[74],"have":[75],"been":[76],"generally":[77],"expected.":[78],"clarifies":[81],"what":[82],"makes":[83,88],"with":[91],"numerical":[92],"examples":[93],"theoretical":[95],"discussions.":[96]},"counts_by_year":[{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":2},{"year":2015,"cited_by_count":4}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}