{"id":"https://openalex.org/W7165806563","doi":"https://doi.org/10.48550/arxiv.2606.24064","title":"Beyond Trajectory Imitation: Strategy-Guided Policy Optimization for LLM Reasoning","display_name":"Beyond Trajectory Imitation: Strategy-Guided Policy Optimization for LLM Reasoning","publication_year":2026,"publication_date":"2026-06-23","ids":{"openalex":"https://openalex.org/W7165806563","doi":"https://doi.org/10.48550/arxiv.2606.24064"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.24064","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24064","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.24064","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139280155","display_name":"Tianyuan Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Tianyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113067526","display_name":"Canbin Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Canbin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139302654","display_name":"Bei Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Bei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139219644","display_name":"Xin Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Xin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139221890","display_name":"Xiaojun Quan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Quan, Xiaojun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139221900","display_name":"Jingang Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jingang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139303287","display_name":"Qifan Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Qifan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.2678999900817871,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.2678999900817871,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.1137000024318695,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.10140000283718109,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.6583999991416931},{"id":"https://openalex.org/keywords/weighting","display_name":"Weighting","score":0.48330000042915344},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.48249998688697815},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.46889999508857727},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.36800000071525574},{"id":"https://openalex.org/keywords/memorization","display_name":"Memorization","score":0.35519999265670776},{"id":"https://openalex.org/keywords/adaptability","display_name":"Adaptability","score":0.35409998893737793},{"id":"https://openalex.org/keywords/trajectory-optimization","display_name":"Trajectory optimization","score":0.33480000495910645}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6898999810218811},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.6583999991416931},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5281999707221985},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.48330000042915344},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.48249998688697815},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.46889999508857727},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4345000088214874},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.36800000071525574},{"id":"https://openalex.org/C30038468","wikidata":"https://www.wikidata.org/wiki/Q4354775","display_name":"Memorization","level":2,"score":0.35519999265670776},{"id":"https://openalex.org/C177606310","wikidata":"https://www.wikidata.org/wiki/Q5674297","display_name":"Adaptability","level":2,"score":0.35409998893737793},{"id":"https://openalex.org/C173246807","wikidata":"https://www.wikidata.org/wiki/Q7833062","display_name":"Trajectory optimization","level":3,"score":0.33480000495910645},{"id":"https://openalex.org/C126388530","wikidata":"https://www.wikidata.org/wiki/Q1131737","display_name":"Imitation","level":2,"score":0.3305000066757202},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.2816999852657318},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2791999876499176},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2777999937534332},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.2768999934196472},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.2734000086784363},{"id":"https://openalex.org/C75553542","wikidata":"https://www.wikidata.org/wiki/Q178161","display_name":"A priori and a posteriori","level":2,"score":0.25540000200271606},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.25459998846054077},{"id":"https://openalex.org/C2778869765","wikidata":"https://www.wikidata.org/wiki/Q6028363","display_name":"Inefficiency","level":2,"score":0.2522999942302704}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.24064","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24064","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.24064","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24064","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Distilling":[0],"reasoning":[1],"capabilities":[2],"from":[3,65],"strong":[4],"to":[5,18,23,42,78,100,126],"weak":[6],"language":[7],"models":[8],"typically":[9],"involves":[10],"imitating":[11],"specific":[12],"solution":[13],"trajectories,":[14],"effectively":[15],"transferring":[16],"what":[17],"answer":[19],"rather":[20,33],"than":[21,34],"how":[22,99],"reason.":[24],"This":[25],"trajectory-level":[26],"imitation":[27,55],"encourages":[28],"memorization":[29],"of":[30,36,82],"instance-specific":[31],"steps":[32],"acquisition":[35],"transferable":[37],"problem-solving":[38],"skills,":[39],"limiting":[40],"generalization":[41],"novel":[43],"problems.":[44],"We":[45],"propose":[46],"Strategy-Guided":[47],"Policy":[48],"Optimization":[49],"(SGPO),":[50],"which":[51],"replaces":[52],"instance-level":[53,129],"trajectory":[54,195],"with":[56,86,119,204],"reusable":[57],"strategy":[58,63,113,199],"distillation.":[59],"SGPO":[60,158],"extracts":[61],"structured":[62],"descriptions":[64],"strong-model":[66],"responses":[67],"and,":[68],"for":[69],"each":[70],"problem,":[71],"constructs":[72],"both":[73],"autonomous":[74,134],"and":[75,87,138,164,197],"strategy-guided":[76],"trajectories":[77],"enable":[79],"direct":[80,194],"comparison":[81],"the":[83,108,116,142,168,175,183],"model's":[84,143],"behavior":[85],"without":[88],"strategic":[89],"guidance.":[90],"The":[91],"framework":[92],"then":[93],"addresses":[94],"two":[95,153],"key":[96],"questions.":[97],"For":[98,124],"distill,":[101,127],"a":[102],"token-level":[103],"forward-KL":[104,184],"objective":[105,185],"selectively":[106],"transfers":[107],"distributional":[109],"shift":[110],"induced":[111],"by":[112,171],"conditioning":[114],"into":[115],"unguided":[117],"policy,":[118],"proximal":[120],"constraints":[121],"ensuring":[122],"stability.":[123],"when":[125,133],"adaptive":[128],"weighting":[130],"strengthens":[131],"guidance":[132],"exploration":[135],"falls":[136],"short":[137],"reduces":[139],"it":[140],"as":[141],"own":[144],"competence":[145],"grows.":[146],"Experiments":[147],"on":[148,178],"four":[149],"mathematical":[150],"benchmarks":[151],"across":[152],"model":[154,206],"families":[155],"show":[156],"that":[157,182,192,198],"consistently":[159],"outperforms":[160,193],"SFT,":[161],"on-policy":[162],"RL,":[163],"hybrid-policy":[165],"baselines,":[166],"improving":[167],"average":[169],"score":[170],"2.2":[172],"points":[173],"over":[174],"strongest":[176],"baseline":[177],"Qwen2.5-7B-Instruct.":[179],"Analysis":[180],"reveals":[181],"provides":[186],"an":[187],"inherently":[188],"selective":[189],"distillation":[190,200],"signal":[191],"imitation,":[196],"exhibits":[201],"complementary":[202],"scaling":[203],"base":[205],"capability.":[207]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-25T00:00:00"}