{"id":"https://openalex.org/W7165135290","doi":"https://doi.org/10.48550/arxiv.2606.18954","title":"GraphPO: Graph-based Policy Optimization for Reasoning Models","display_name":"GraphPO: Graph-based Policy Optimization for Reasoning Models","publication_year":2026,"publication_date":"2026-06-17","ids":{"openalex":"https://openalex.org/W7165135290","doi":"https://doi.org/10.48550/arxiv.2606.18954"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.18954","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18954","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.18954","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114968374","display_name":"Yuliang Zhan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhan, Yuliang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004010230","display_name":"X W Tang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Xinyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138923099","display_name":"Jian Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Jian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138891500","display_name":"Dandan Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Dandan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041226733","display_name":"Weilong Chai","orcid":"https://orcid.org/0009-0003-0038-653X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chai, Weilong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138882691","display_name":"Jingdong Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jingdong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138843065","display_name":"Jun Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Jun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138861000","display_name":"Ge Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Ge","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039068400","display_name":"Wenyue Tang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Wenyue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138867527","display_name":"Hao Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.241799995303154,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.241799995303154,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.19709999859333038,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.17839999496936798,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.6019999980926514},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5863000154495239},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.5139999985694885},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.4925000071525574},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.48510000109672546},{"id":"https://openalex.org/keywords/equivalence","display_name":"Equivalence (formal languages)","score":0.47920000553131104},{"id":"https://openalex.org/keywords/prefix","display_name":"Prefix","score":0.43299999833106995},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.3955000042915344},{"id":"https://openalex.org/keywords/commonsense-reasoning","display_name":"Commonsense reasoning","score":0.35989999771118164}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7321000099182129},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6019999980926514},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5863000154495239},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5139999985694885},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.4925000071525574},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.48510000109672546},{"id":"https://openalex.org/C2780069185","wikidata":"https://www.wikidata.org/wiki/Q7977945","display_name":"Equivalence (formal languages)","level":2,"score":0.47920000553131104},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4357999861240387},{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.43299999833106995},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.4106000065803528},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.3955000042915344},{"id":"https://openalex.org/C193221554","wikidata":"https://www.wikidata.org/wiki/Q5153664","display_name":"Commonsense reasoning","level":2,"score":0.35989999771118164},{"id":"https://openalex.org/C159032336","wikidata":"https://www.wikidata.org/wiki/Q2488768","display_name":"Non-monotonic logic","level":2,"score":0.32580000162124634},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.32429999113082886},{"id":"https://openalex.org/C89288958","wikidata":"https://www.wikidata.org/wiki/Q7301504","display_name":"Reasoning system","level":2,"score":0.31790000200271606},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.31380000710487366},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3109999895095825},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.2969000041484833},{"id":"https://openalex.org/C115086926","wikidata":"https://www.wikidata.org/wiki/Q17004651","display_name":"Causal reasoning","level":3,"score":0.29179999232292175},{"id":"https://openalex.org/C83725634","wikidata":"https://www.wikidata.org/wiki/Q7268699","display_name":"Qualitative reasoning","level":2,"score":0.28780001401901245},{"id":"https://openalex.org/C44616089","wikidata":"https://www.wikidata.org/wiki/Q30158686","display_name":"Constraint satisfaction","level":3,"score":0.2854999899864197},{"id":"https://openalex.org/C103057564","wikidata":"https://www.wikidata.org/wiki/Q4751139","display_name":"Analytic reasoning","level":3,"score":0.265500009059906},{"id":"https://openalex.org/C74197172","wikidata":"https://www.wikidata.org/wiki/Q1195339","display_name":"Directed acyclic graph","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.2614000141620636},{"id":"https://openalex.org/C195344581","wikidata":"https://www.wikidata.org/wiki/Q2555318","display_name":"Automated reasoning","level":2,"score":0.2517000138759613}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.18954","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18954","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.18954","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18954","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"Learning":[1],"with":[2,150,243],"Verifiable":[3],"Rewards":[4],"(RLVR)":[5],"has":[6,34],"become":[7],"a":[8,138,146],"standard":[9],"paradigm":[10,33],"for":[11],"enhancing":[12],"the":[13,26,76,160,244],"capability":[14],"of":[15],"large":[16],"reasoning":[17,44,95,151,161,169,222,229],"models.":[18],"RLVR":[19],"typically":[20],"samples":[21],"responses":[22,39],"independently":[23,38],"and":[24,49,72,101,111,155,179,197,220,230,240],"optimizes":[25],"policy":[27],"using":[28],"from":[29,75,159,183,211],"final":[30],"answers.":[31],"This":[32],"two":[35],"limitations.":[36],"First,":[37],"often":[40],"contain":[41],"similar":[42,94,103],"intermediate":[43],"steps,":[45],"causing":[46],"redundant":[47,184],"exploration":[48],"wasted":[50],"computation.":[51],"Second,":[52],"sparse":[53],"final-answer":[54],"rewards":[55],"make":[56],"it":[57],"hard":[58],"to":[59,79,122,176,186,194,200],"identify":[60],"useful":[61],"steps.":[62],"Tree-based":[63],"methods":[64,107],"partly":[65],"address":[66,129],"this":[67,130],"problem":[68],"by":[69],"sharing":[70],"prefixes":[71],"comparing":[73],"branches":[74,85,92],"same":[77,245],"prefix":[78],"provide":[80],"fine-grained":[81],"signals.":[82],"However,":[83],"tree":[84],"are":[86],"still":[87],"expanded":[88],"independently.":[89],"When":[90],"different":[91],"reach":[93],"states,":[96],"they":[97],"cannot":[98],"share":[99,177],"information":[100],"repeat":[102],"exploration.":[104,188],"Moreover,":[105],"tree-based":[106,241],"ignore":[108],"such":[109],"dispersion":[110],"only":[112],"perform":[113],"local":[114],"comparisons":[115],"within":[116],"separate":[117],"branches,":[118],"which":[119],"can":[120],"lead":[121],"higher":[123],"variance":[124,219],"in":[125],"advantage":[126],"estimation.":[127],"To":[128],"challenge,":[131],"we":[132,190],"propose":[133],"GraphPO":[134,165,216,236],"(Graph-based":[135],"Policy":[136],"Optimization),":[137],"novel":[139],"RL":[140],"framework":[141],"that":[142,215,235],"represents":[143],"rollouts":[144],"as":[145,153,163],"directed":[147],"acyclic":[148],"graph,":[149],"steps":[152],"edges":[154,196],"semantic":[156],"states":[157],"summarized":[158],"paths":[162,170],"nodes.":[164],"merges":[166],"semantically":[167],"equivalent":[168],"into":[171],"equivalence":[172],"classes,":[173],"allowing":[174],"them":[175],"suffixes":[178],"reallocating":[180],"budget":[181],"away":[182],"expansions":[185],"diverse":[187],"Furthermore,":[189],"assign":[191],"efficiency":[192,206],"advantages":[193,199],"incoming":[195],"correctness":[198],"outgoing":[201],"edges,":[202],"thereby":[203],"improving":[204],"inference":[205],"while":[207],"deriving":[208],"process":[209],"supervision":[210],"outcome.":[212],"Theory":[213],"shows":[214],"reduces":[217],"advantage-estimation":[218],"enhances":[221],"efficiency.":[223],"Experiments":[224],"on":[225],"three":[226],"LLMs":[227],"across":[228],"agentic":[231],"search":[232],"benchmarks":[233],"show":[234],"consistently":[237],"outperforms":[238],"chain-":[239],"baselines":[242],"token":[246],"budgets":[247],"or":[248],"response":[249],"budgets.":[250]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-19T00:00:00"}