{"id":"https://openalex.org/W7164146714","doi":"https://doi.org/10.48550/arxiv.2606.11087","title":"Test-Time Gradient Guidance of Flow Policies in Reinforcement Learning","display_name":"Test-Time Gradient Guidance of Flow Policies in Reinforcement Learning","publication_year":2026,"publication_date":"2026-06-09","ids":{"openalex":"https://openalex.org/W7164146714","doi":"https://doi.org/10.48550/arxiv.2606.11087"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.11087","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.11087","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.11087","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138288001","display_name":"Zhiyuan Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Zhiyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138326894","display_name":"Andy Peng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Andy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138368172","display_name":"Charles Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Charles","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138316811","display_name":"Qiyang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Qiyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062608295","display_name":"Tobias Springenberg","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Springenberg, Tobias","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053366330","display_name":"Kevin Frans","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Frans, Kevin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138385740","display_name":"Sergey Levine","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Levine, Sergey","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8227999806404114,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8227999806404114,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.08780000358819962,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10784","display_name":"Muscle activation and electromyography studies","score":0.01080000028014183,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.8676999807357788},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.5503000020980835},{"id":"https://openalex.org/keywords/imitation","display_name":"Imitation","score":0.5127999782562256},{"id":"https://openalex.org/keywords/supervised-learning","display_name":"Supervised learning","score":0.4180000126361847},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.40630000829696655},{"id":"https://openalex.org/keywords/flow","display_name":"Flow (mathematics)","score":0.38040000200271606},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.36250001192092896},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.349700003862381}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8676999807357788},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7919999957084656},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.5503000020980835},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5436000227928162},{"id":"https://openalex.org/C126388530","wikidata":"https://www.wikidata.org/wiki/Q1131737","display_name":"Imitation","level":2,"score":0.5127999782562256},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4512999951839447},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.4180000126361847},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.40630000829696655},{"id":"https://openalex.org/C38349280","wikidata":"https://www.wikidata.org/wiki/Q1434290","display_name":"Flow (mathematics)","level":2,"score":0.38040000200271606},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.36250001192092896},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.349700003862381},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.3294999897480011},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.3246000111103058},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.3156000077724457},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.3100000023841858},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.30379998683929443},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3028999865055084},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.30250000953674316},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.2962000072002411},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.2906999886035919},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.2770000100135803},{"id":"https://openalex.org/C186766456","wikidata":"https://www.wikidata.org/wiki/Q612457","display_name":"Flow control (data)","level":2,"score":0.272599995136261},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.2700999975204468},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.26019999384880066}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.11087","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.11087","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.11087","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.11087","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.550757110118866,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Expressive":[0],"continuous":[1],"control":[2],"policies,":[3],"such":[4],"as":[5],"diffusion":[6],"and":[7,22,70,139,175,184,215],"flow":[8,131],"models,":[9],"form":[10],"the":[11,34,78,149,154,207],"backbone":[12],"of":[13,80,209],"recent":[14],"advances":[15],"in":[16,33],"scaling":[17,201],"imitation":[18,36],"learning":[19,37,43],"for":[20,46],"simulated":[21],"real":[23],"robot":[24],"control.":[25],"While":[26],"they":[27],"are":[28],"known":[29],"to":[30,152,157,195],"scale":[31],"stably":[32],"supervised":[35,92],"setting,":[38],"incorporating":[39],"them":[40],"into":[41],"reinforcement":[42],"(RL)":[44],"pipelines":[45],"policy":[47,83,93,118,132,156,164],"improvement":[48,84],"has":[49],"proven":[50],"more":[51],"difficult.":[52],"It":[53],"often":[54],"requires":[55],"specialized":[56],"training":[57,94],"objectives":[58],"or":[59],"backpropagating":[60],"through":[61],"denoising":[62],"processes,":[63],"which":[64,101],"cause":[65],"well-known":[66],"issues":[67],"with":[68,180,187,202,220],"stability":[69],"affect":[71],"scalability.":[72],"In":[73],"this":[74,106],"paper":[75],"we":[76,108],"study":[77],"question":[79],"whether":[81],"simple":[82],"schemes":[85],"at":[86,121,145],"test":[87,122,146],"time":[88],"alone,":[89],"leaving":[90],"stable":[91],"intact,":[95],"can":[96],"be":[97],"a":[98,129,134,140,213],"competitive":[99,186],"alternative":[100,217],"sidesteps":[102],"these":[103],"issues.":[104],"To":[105],"end,":[107],"propose":[109],"QGF":[110,124,167],"(Q-Guided":[111],"Flow),":[112],"an":[113],"RL":[114,171,178,218],"algorithm":[115,219],"that":[116],"performs":[117],"optimization":[119],"entirely":[120],"time.":[123],"works":[125],"by":[126,205],"pre-training":[127],"both":[128],"reference":[130,155],"(via":[133],"standard":[135],"behavioral":[136],"cloning":[137],"objective)":[138],"value":[141,150],"function":[142],"critic":[143],"and,":[144],"time,":[147],"using":[148],"gradient":[151],"guide":[153],"generate":[158],"higher-value":[159],"actions":[160],"without":[161],"any":[162],"additional":[163],"learning.":[165],"Empirically,":[166],"outperforms":[168],"prior":[169],"test-time":[170],"methods":[172],"on":[173],"single-task":[174],"goal-conditioned":[176],"offline":[177],"benchmarks":[179],"high-dimensional":[181],"action":[182],"spaces,":[183],"is":[185],"state-of-the-art":[188],"training-time":[189],"algorithms":[190],"while":[191],"being":[192],"much":[193],"cheaper":[194],"run.":[196],"Moreover,":[197],"it":[198],"exhibits":[199],"favorable":[200],"model":[203],"size":[204],"avoiding":[206],"instability":[208],"actor-critic":[210],"training,":[211],"offering":[212],"practical":[214],"effective":[216],"expressive":[221],"policies.":[222]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-11T00:00:00"}