{"id":"https://openalex.org/W7164124608","doi":"https://doi.org/10.48550/arxiv.2606.09887","title":"SocraticPO: Policy Optimization via Interactive Guidance","display_name":"SocraticPO: Policy Optimization via Interactive Guidance","publication_year":2026,"publication_date":"2026-06-03","ids":{"openalex":"https://openalex.org/W7164124608","doi":"https://doi.org/10.48550/arxiv.2606.09887"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.09887","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.09887","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.09887","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138349064","display_name":"Zirui Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Zirui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138311310","display_name":"Jie Ouyang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ouyang, Jie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138337839","display_name":"Qi Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Qi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138303440","display_name":"Xianquan Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xianquan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138313090","display_name":"Jiayu Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jiayu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114133092","display_name":"Tingyue Pan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pan, Tingyue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102791982","display_name":"Qingchuan Li","orcid":"https://orcid.org/0000-0001-9915-2589"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Qingchuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054303589","display_name":"Jing Sha","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sha, Jing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138344595","display_name":"Zhenya Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Zhenya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138346688","display_name":"Shijin Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Shijin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138366448","display_name":"Enhong Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Enhong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.37940001487731934,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.37940001487731934,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.13920000195503235,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.06310000270605087,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6984000205993652},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5328999757766724},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5177000164985657},{"id":"https://openalex.org/keywords/intervention","display_name":"Intervention (counseling)","score":0.3855000138282776},{"id":"https://openalex.org/keywords/path","display_name":"Path (computing)","score":0.35109999775886536},{"id":"https://openalex.org/keywords/policy-learning","display_name":"Policy learning","score":0.34529998898506165}],"concepts":[{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6984000205993652},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6592000126838684},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5328999757766724},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5177000164985657},{"id":"https://openalex.org/C2780665704","wikidata":"https://www.wikidata.org/wiki/Q959298","display_name":"Intervention (counseling)","level":2,"score":0.3855000138282776},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.35109999775886536},{"id":"https://openalex.org/C2779436431","wikidata":"https://www.wikidata.org/wiki/Q30672407","display_name":"Policy learning","level":2,"score":0.34529998898506165},{"id":"https://openalex.org/C174998907","wikidata":"https://www.wikidata.org/wiki/Q357662","display_name":"Work in process","level":2,"score":0.34119999408721924},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.3361999988555908},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.3260999917984009},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3172999918460846},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.30410000681877136},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2969000041484833},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2964000105857849},{"id":"https://openalex.org/C2779110517","wikidata":"https://www.wikidata.org/wiki/Q1240788","display_name":"Supervisor","level":2,"score":0.29030001163482666},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2775999903678894},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.27709999680519104},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.2703999876976013},{"id":"https://openalex.org/C127729010","wikidata":"https://www.wikidata.org/wiki/Q60165","display_name":"Dynamic inconsistency","level":2,"score":0.2662999927997589}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.09887","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.09887","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.09887","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.09887","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.6941970586776733,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1,39],"(RL)":[2],"for":[3],"large":[4],"language":[5],"models":[6,161],"usually":[7],"supervises":[8],"reasoning":[9,173],"with":[10,56,96,196],"scalar":[11],"outcome":[12],"rewards,":[13,108],"such":[14,144],"as":[15,116,145],"binary":[16],"correctness.":[17],"Such":[18],"rewards":[19],"provide":[20],"an":[21],"optimization":[22],"direction":[23],"but":[24],"rarely":[25],"explain":[26],"how":[27],"a":[28,49,72,117],"model":[29],"should":[30],"revise":[31],"its":[32],"mistaken":[33],"reasoning,":[34],"which":[35,83],"can":[36,137,156],"encourage":[37],"shortcut":[38],"and":[40,77,182,191],"brittle":[41],"policies.":[42],"We":[43],"propose":[44],"\\textbf{SocraticPO}":[45],"(Socratic":[46],"Policy":[47],"Optimization),":[48],"policy-optimization":[50],"framework":[51],"that":[52,187],"augments":[53],"RL":[54,181],"rollouts":[55],"Socratic-style":[57],"natural-language":[58],"guidance.":[59],"During":[60],"rollout,":[61],"the":[62,68,75,84,88,110,126,131,149],"student":[63,85],"first":[64],"answers":[65,100],"independently;":[66],"if":[67],"answer":[69],"is":[70,94],"incorrect,":[71],"teacher":[73,103,114,150,160],"diagnoses":[74],"attempt":[76],"provides":[78,151],"concise":[79],"corrective":[80],"guidance,":[81,154],"after":[82,102],"continues":[86],"under":[87],"expanded":[89],"context.":[90],"Crucially,":[91],"this":[92],"guidance":[93,190],"paired":[95],"reward":[97,192,197],"decay:":[98],"correct":[99],"obtained":[101],"intervention":[104],"only":[105,124,152],"receive":[106],"decayed":[107],"preventing":[109],"policy":[111],"from":[112,175],"treating":[113],"help":[115],"free":[118],"path":[119],"to":[120,165],"reward.":[121],"Since":[122],"SocraticPO":[123,155,177],"modifies":[125],"rollout":[127],"process":[128],"while":[129],"leaving":[130],"standard":[132],"expected-reward":[133],"objective":[134],"intact,":[135],"it":[136],"be":[138],"plugged":[139],"into":[140],"existing":[141],"policy-gradient":[142],"backends":[143],"Reinforce++.":[146],"Moreover,":[147],"because":[148],"text-level":[153],"leverage":[157],"stronger":[158],"black-box":[159],"without":[162],"requiring":[163],"access":[164],"logits":[166],"or":[167],"distribution":[168],"matching.":[169],"On":[170],"undergraduate-level":[171],"scientific":[172],"benchmarks":[174],"SciKnowEval,":[176],"improves":[178],"over":[179],"strong":[180],"self-distillation":[183],"baselines.":[184],"Ablations":[185],"show":[186],"both":[188],"targeted":[189],"decay":[193,198],"are":[194],"necessary,":[195],"mitigating":[199],"reliance":[200],"on":[201],"assisted":[202],"correction.":[203]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-11T00:00:00"}