{"id":"https://openalex.org/W7225149015","doi":"https://doi.org/10.48550/arxiv.2611.02414","title":"SGR-OPSD: Learning from Subgoal Plans with Graph-Regularized Credit Assignment for Agentic Reinforcement Learning","display_name":"SGR-OPSD: Learning from Subgoal Plans with Graph-Regularized Credit Assignment for Agentic Reinforcement Learning","publication_year":2026,"publication_date":"2026-11-02","ids":{"openalex":"https://openalex.org/W7225149015","doi":"https://doi.org/10.48550/arxiv.2611.02414"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2611.02414","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.02414","pdf_url":"https://arxiv.org/pdf/2611.02414","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2611.02414","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5160515894","display_name":"Shu Liu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405259410","display_name":"Meituan (China)","ror":"https://ror.org/0272kpg92","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405259410"]},{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Shu","raw_affiliation_strings":["MeiTuan","School of Computer Science and Technology , East China Normal University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MeiTuan","institution_ids":["https://openalex.org/I4405259410"]},{"raw_affiliation_string":"School of Computer Science and Technology , East China Normal University","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020229041","display_name":"Chong Xin Yang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405259410","display_name":"Meituan (China)","ror":"https://ror.org/0272kpg92","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405259410"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chong, Yang","raw_affiliation_strings":["MeiTuan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MeiTuan","institution_ids":["https://openalex.org/I4405259410"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160470995","display_name":"WenShi Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405259410","display_name":"Meituan (China)","ror":"https://ror.org/0272kpg92","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405259410"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen, WenShi","raw_affiliation_strings":["MeiTuan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MeiTuan","institution_ids":["https://openalex.org/I4405259410"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057508424","display_name":"Man Lan","orcid":"https://orcid.org/0000-0002-1423-1286","observed_orcids":["https://orcid.org/0000-0002-1423-1286"]},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lan, Man","raw_affiliation_strings":["School of Computer Science and Technology , East China Normal University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology , East China Normal University","institution_ids":["https://openalex.org/I66867065"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.0019000000320374966,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/agentic-reinforcement-learning","display_name":"agentic reinforcement learning","score":0.9480000138282776},{"id":"https://openalex.org/keywords/privileged-information","display_name":"privileged information","score":0.4490000009536743},{"id":"https://openalex.org/keywords/credit-assignment","display_name":"credit assignment","score":0.7139999866485596},{"id":"https://openalex.org/keywords/graph-regularization","display_name":"graph regularization","score":0.777999997138977},{"id":"https://openalex.org/keywords/kl-regularization","display_name":"KL regularization","score":0.6690000295639038},{"id":"https://openalex.org/keywords/alfworld","display_name":"ALFWorld","score":0.6740000247955322},{"id":"https://openalex.org/keywords/webshop","display_name":"WebShop","score":0.8360000252723694}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7050999999046326},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.5270000100135803},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4927999973297119},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4909999966621399},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48260000348091125},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4781999886035919},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.47620001435279846},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.4587000012397766},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.4377000033855438},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.42309999465942383},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.38850000500679016},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3700000047683716},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.32589998841285706},{"id":"https://openalex.org/C113336015","wikidata":"https://www.wikidata.org/wiki/Q574010","display_name":"Complete information","level":2,"score":0.3131999969482422},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.3098999857902527},{"id":"https://openalex.org/C79416737","wikidata":"https://www.wikidata.org/wiki/Q2305519","display_name":"Social learning","level":2,"score":0.3050999939441681},{"id":"https://openalex.org/C2779110102","wikidata":"https://www.wikidata.org/wiki/Q1323737","display_name":"Revealed preference","level":2,"score":0.2980000078678131},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.28850001096725464},{"id":"https://openalex.org/C188116033","wikidata":"https://www.wikidata.org/wiki/Q2664563","display_name":"Q-learning","level":3,"score":0.2854999899864197},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.28189998865127563},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.27730000019073486},{"id":"https://openalex.org/C146380142","wikidata":"https://www.wikidata.org/wiki/Q1137726","display_name":"Directed graph","level":2,"score":0.2705000042915344},{"id":"https://openalex.org/C2777138686","wikidata":"https://www.wikidata.org/wiki/Q1787103","display_name":"Credit score","level":2,"score":0.26249998807907104},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.2612000107765198}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2611.02414","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.02414","pdf_url":"https://arxiv.org/pdf/2611.02414","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2611.02414","doi":"https://doi.org/10.48550/arxiv.2611.02414","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.02414","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2611.02414","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.02414","pdf_url":"https://arxiv.org/pdf/2611.02414","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7225149015.pdf","grobid_xml":"https://content.openalex.org/works/W7225149015.grobid-xml"},"referenced_works_count":15,"referenced_works":["https://openalex.org/W3126325318","https://openalex.org/W4391631327","https://openalex.org/W4405903187","https://openalex.org/W4417422257","https://openalex.org/W7125861034","https://openalex.org/W7133227044","https://openalex.org/W7150828934","https://openalex.org/W7154299763","https://openalex.org/W7161258139","https://openalex.org/W7172399155","https://openalex.org/W7201837771","https://openalex.org/W7203675926","https://openalex.org/W7204891669","https://openalex.org/W7214799011","https://openalex.org/W7214805187"],"related_works":["https://openalex.org/W7201837771","https://openalex.org/W7202134906","https://openalex.org/W7216169477","https://openalex.org/W7162494674","https://openalex.org/W7214805187","https://openalex.org/W7221810633","https://openalex.org/W7224931313","https://openalex.org/W7124347845","https://openalex.org/W7214775583","https://openalex.org/W7214358361"],"abstract_inverted_index":{"OPSD":[0,43],"provides":[1],"dense":[2,66],"supervision":[3,50],"for":[4,54],"agentic":[5],"reinforcement":[6],"learning":[7],"through":[8],"privileged":[9,14],"information":[10,15],"feedback.":[11],"However,":[12],"existing":[13],"often":[16],"lacks":[17],"explicit":[18],"task-progression":[19],"planning,":[20],"while":[21],"teacher-student":[22],"log-probability":[23],"gaps":[24],"reflect":[25],"shifts":[26],"in":[27,82],"model":[28,126],"preference":[29],"rather":[30],"than":[31],"true":[32],"task":[33],"progress.":[34],"To":[35,79],"address":[36],"these":[37,170],"limitations,":[38],"we":[39,85],"propose":[40],"Subgoal-Guided":[41],"Graph-Regularized":[42],"(SGR-OPSD),":[44],"a":[45,75,87],"framework":[46],"that":[47,91,112,160],"combines":[48],"subgoal":[49,62,161],"with":[51,155],"interaction-derived":[52],"evidence":[53],"turn-level":[55],"credit":[56,81,116],"assignment.":[57],"Specifically,":[58],"SGR-OPSD":[59,131],"uses":[60],"LLM-generated":[61],"plans":[63],"to":[64,109,146,169],"provide":[65],"action-level":[67],"guidance,":[68],"converting":[69],"them":[70],"into":[71],"advantage":[72],"weights":[73],"via":[74],"constrained,":[76],"KL-regularized":[77],"objective.":[78],"ground":[80],"observed":[83],"outcomes,":[84],"construct":[86],"Subgoal":[88],"Progress":[89],"Graph":[90],"estimates":[92],"empirical":[93,102],"returns":[94,104],"from":[95],"sampled":[96],"transition":[97],"frequencies.":[98],"The":[99],"graph":[100],"compares":[101],"action":[103],"against":[105],"the":[106,114],"state":[107],"average":[108],"derive":[110],"preferences":[111],"calibrate":[113],"teacher's":[115],"weights.":[117],"Evaluated":[118],"on":[119,153],"ALFWorld,":[120],"WebShop,":[121],"and":[122,129,135,150,163],"Search-QA":[123],"across":[124],"two":[125],"scales":[127],"(Qwen2.5-3B":[128],"7B),":[130],"consistently":[132],"outperforms":[133],"GRPO":[134,143],"strong":[136],"self-distillation":[137],"baselines,":[138],"improving":[139],"success":[140],"rates":[141],"over":[142],"by":[144],"up":[145],"7.4":[147],"percentage":[148],"points":[149],"reaching":[151],"85.8%":[152],"ALFWorld":[154],"Qwen2.5-7B.":[156],"Ablation":[157],"studies":[158],"confirm":[159],"guidance":[162],"graph-based":[164],"weight":[165],"calibration":[166],"are":[167],"key":[168],"performance":[171],"gains.":[172]},"counts_by_year":[],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2026-11-04T00:00:00"}