{"id":"https://openalex.org/W7128240039","doi":"https://doi.org/10.1109/tnnls.2025.3641531","title":"Enhancing PPO With Trajectory-Aware Hybrid Policies","display_name":"Enhancing PPO With Trajectory-Aware Hybrid Policies","publication_year":2026,"publication_date":"2026-02-06","ids":{"openalex":"https://openalex.org/W7128240039","doi":"https://doi.org/10.1109/tnnls.2025.3641531","pmid":"https://pubmed.ncbi.nlm.nih.gov/41650423"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2025.3641531","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2025.3641531","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010702633","display_name":"Qisai Liu","orcid":"https://orcid.org/0000-0002-9553-0789"},"institutions":[{"id":"https://openalex.org/I173911158","display_name":"Iowa State University","ror":"https://ror.org/04rswrd78","country_code":"US","type":"education","lineage":["https://openalex.org/I173911158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qisai Liu","raw_affiliation_strings":["Department of Mechanical Engineering, Iowa State University, Ames, IA, USA"],"raw_orcid":"https://orcid.org/0000-0002-9553-0789","affiliations":[{"raw_affiliation_string":"Department of Mechanical Engineering, Iowa State University, Ames, IA, USA","institution_ids":["https://openalex.org/I173911158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031212270","display_name":"Zhanhong Jiang","orcid":"https://orcid.org/0000-0001-5363-7898"},"institutions":[{"id":"https://openalex.org/I173911158","display_name":"Iowa State University","ror":"https://ror.org/04rswrd78","country_code":"US","type":"education","lineage":["https://openalex.org/I173911158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhanhong Jiang","raw_affiliation_strings":["Translational AI Center, Iowa State University, Ames, IA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Translational AI Center, Iowa State University, Ames, IA, USA","institution_ids":["https://openalex.org/I173911158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101767629","display_name":"Hsin-Jung Yang","orcid":"https://orcid.org/0000-0001-8395-5993"},"institutions":[{"id":"https://openalex.org/I173911158","display_name":"Iowa State University","ror":"https://ror.org/04rswrd78","country_code":"US","type":"education","lineage":["https://openalex.org/I173911158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hsin-Jung Yang","raw_affiliation_strings":["Department of Mechanical Engineering, Iowa State University, Ames, IA, USA"],"raw_orcid":"https://orcid.org/0000-0001-8395-5993","affiliations":[{"raw_affiliation_string":"Department of Mechanical Engineering, Iowa State University, Ames, IA, USA","institution_ids":["https://openalex.org/I173911158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125328295","display_name":"Mahsa Khosravi","orcid":null},"institutions":[{"id":"https://openalex.org/I173911158","display_name":"Iowa State University","ror":"https://ror.org/04rswrd78","country_code":"US","type":"education","lineage":["https://openalex.org/I173911158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mahsa Khosravi","raw_affiliation_strings":["Department of Industrial and Manufacturing Systems Engineering, Iowa State University, Ames, IA, USA"],"raw_orcid":"https://orcid.org/0009-0009-6592-1111","affiliations":[{"raw_affiliation_string":"Department of Industrial and Manufacturing Systems Engineering, Iowa State University, Ames, IA, USA","institution_ids":["https://openalex.org/I173911158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009697582","display_name":"Joshua R. Waite","orcid":"https://orcid.org/0000-0003-4187-6167"},"institutions":[{"id":"https://openalex.org/I173911158","display_name":"Iowa State University","ror":"https://ror.org/04rswrd78","country_code":"US","type":"education","lineage":["https://openalex.org/I173911158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joshua R. Waite","raw_affiliation_strings":["Translational AI Center, Iowa State University, Ames, IA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Translational AI Center, Iowa State University, Ames, IA, USA","institution_ids":["https://openalex.org/I173911158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5125289966","display_name":"Soumik Sarkar","orcid":null},"institutions":[{"id":"https://openalex.org/I173911158","display_name":"Iowa State University","ror":"https://ror.org/04rswrd78","country_code":"US","type":"education","lineage":["https://openalex.org/I173911158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Soumik Sarkar","raw_affiliation_strings":["Department of Mechanical Engineering, Iowa State University, Ames, IA, USA"],"raw_orcid":"https://orcid.org/0000-0002-6775-9199","affiliations":[{"raw_affiliation_string":"Department of Mechanical Engineering, Iowa State University, Ames, IA, USA","institution_ids":["https://openalex.org/I173911158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I173911158"],"apc_list":null,"apc_paid":null,"fwci":12.1051,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.96663466,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"37","issue":"6","first_page":"2869","last_page":"2883"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8072999715805054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8072999715805054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.04580000042915344,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.0348999984562397,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.6202999949455261},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5896000266075134},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.5823000073432922},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.5491999983787537},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.526199996471405},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.4562999904155731},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.40939998626708984}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7366999983787537},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.6202999949455261},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5896000266075134},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.5823000073432922},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5491999983787537},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.526199996471405},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.4562999904155731},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.40939998626708984},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.40720000863075256},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.38929998874664307},{"id":"https://openalex.org/C2778915421","wikidata":"https://www.wikidata.org/wiki/Q3643177","display_name":"Performance improvement","level":2,"score":0.3321000039577484},{"id":"https://openalex.org/C145018004","wikidata":"https://www.wikidata.org/wiki/Q4985944","display_name":"Buffer (optical fiber)","level":2,"score":0.2815999984741211},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27160000801086426},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.26820001006126404},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.26489999890327454},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.26440000534057617},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.2599000036716461},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.25600001215934753},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2540000081062317},{"id":"https://openalex.org/C2780440489","wikidata":"https://www.wikidata.org/wiki/Q5227278","display_name":"Data-driven","level":2,"score":0.25270000100135803}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2025.3641531","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2025.3641531","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:41650423","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41650423","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.523431122303009}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Proximal":[0],"policy":[1,35,125,154],"optimization":[2],"(PPO)":[3],"is":[4,120,162,177],"one":[5],"of":[6,71,105,138],"the":[7,78,81,93,98,106,109,118,124,130,139,153,158],"most":[8,140],"popular":[9],"state-of-the-art":[10],"on-policy":[11,49],"algorithms":[12,169],"that":[13],"has":[14],"become":[15],"a":[16,57,63],"standard":[17],"baseline":[18,168],"in":[19,25,48,145],"modern":[20],"reinforcement":[21],"learning":[22],"with":[23,33,108],"applications":[24],"numerous":[26],"fields.":[27],"Though":[28],"it":[29],"delivers":[30],"stable":[31],"performance":[32,143],"theoretical":[34],"improvement":[36,155],"guarantees,":[37],"high":[38,41],"variance":[39,148],"and":[40,112,164],"sample":[42],"complexity":[43],"still":[44],"remain":[45],"critical":[46],"challenges":[47],"algorithms.":[50],"To":[51],"alleviate":[52],"these":[53],"issues,":[54],"we":[55],"propose":[56],"hybrid-policy":[58],"PPO":[59],"(HP3O),":[60],"which":[61],"utilizes":[62],"trajectory":[64,107],"replay":[65],"buffer":[66,79,119],"to":[67,90,96,132],"make":[68],"efficient":[69],"use":[70],"trajectories":[72,95],"generated":[73],"by":[74],"recent":[75,94,141],"policies.":[76],"Particularly,":[77],"applies":[80],"\"first":[82],"in,":[83],"first":[84],"out\"":[85],"(FIFO)":[86],"strategy":[87,128],"so":[88],"as":[89],"keep":[91],"only":[92],"attenuate":[97],"data":[99],"distribution":[100],"drift.":[101],"A":[102],"batch":[103],"consisting":[104],"best":[110,142],"return":[111],"other":[113],"randomly":[114],"sampled":[115],"ones":[116],"from":[117],"used":[121],"for":[122,157],"updating":[123],"networks.":[126],"The":[127],"helps":[129],"agent":[131],"improve":[133],"its":[134],"capability":[135],"on":[136],"top":[137],"and,":[144],"turn,":[146],"reduce":[147],"empirically.":[149],"We":[150],"theoretically":[151],"construct":[152],"guarantees":[156],"proposed":[159],"algorithm.":[160],"HP3O":[161],"validated":[163],"compared":[165],"against":[166],"several":[167],"using":[170],"multiple":[171],"continuous":[172],"control":[173],"environments.":[174],"Our":[175],"code":[176],"available":[178],"at":[179],"https://anonymous.4open.science/r/HP30-EB61/HP3O_train.py.":[180]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-06-04T06:14:18.571774","created_date":"2026-02-08T00:00:00"}