{"id":"https://openalex.org/W7164994288","doi":"https://doi.org/10.48550/arxiv.2606.17890","title":"Dynamic Rollout Editing for Reducing Overthinking in RL-Trained Reasoning Models","display_name":"Dynamic Rollout Editing for Reducing Overthinking in RL-Trained Reasoning Models","publication_year":2026,"publication_date":"2026-06-16","ids":{"openalex":"https://openalex.org/W7164994288","doi":"https://doi.org/10.48550/arxiv.2606.17890"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.17890","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.17890","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.17890","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138828258","display_name":"Zihao Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Zihao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138772216","display_name":"Wenjie Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Wenjie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138823856","display_name":"Liang Pang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pang, Liang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138781108","display_name":"Jingcheng Deng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deng, Jingcheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053837667","display_name":"S Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Shicheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138760567","display_name":"Shasha Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Shasha","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138812753","display_name":"Zenghao Duan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Duan, Zenghao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138788454","display_name":"Jiahao Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jiahao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138762103","display_name":"Jingang Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jingang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138785007","display_name":"Huawei Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Huawei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138768430","display_name":"Xueqi Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Xueqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.25999999046325684,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.25999999046325684,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1185000017285347,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0885000005364418,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.6313999891281128},{"id":"https://openalex.org/keywords/continuation","display_name":"Continuation","score":0.6218000054359436},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5498999953269958},{"id":"https://openalex.org/keywords/framing","display_name":"Framing (construction)","score":0.5476999878883362},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.4115000069141388},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.36910000443458557},{"id":"https://openalex.org/keywords/phenomenon","display_name":"Phenomenon","score":0.3662000000476837},{"id":"https://openalex.org/keywords/intervention","display_name":"Intervention (counseling)","score":0.36500000953674316}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7207000255584717},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.6313999891281128},{"id":"https://openalex.org/C88626702","wikidata":"https://www.wikidata.org/wiki/Q1128903","display_name":"Continuation","level":2,"score":0.6218000054359436},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5498999953269958},{"id":"https://openalex.org/C169087156","wikidata":"https://www.wikidata.org/wiki/Q2131593","display_name":"Framing (construction)","level":2,"score":0.5476999878883362},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.4115000069141388},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.36910000443458557},{"id":"https://openalex.org/C50335755","wikidata":"https://www.wikidata.org/wiki/Q483247","display_name":"Phenomenon","level":2,"score":0.3662000000476837},{"id":"https://openalex.org/C2780665704","wikidata":"https://www.wikidata.org/wiki/Q959298","display_name":"Intervention (counseling)","level":2,"score":0.36500000953674316},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36090001463890076},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.3434999883174896},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.3287999927997589},{"id":"https://openalex.org/C147297375","wikidata":"https://www.wikidata.org/wiki/Q6674930","display_name":"Look-ahead","level":2,"score":0.31859999895095825},{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.30709999799728394},{"id":"https://openalex.org/C5065155","wikidata":"https://www.wikidata.org/wiki/Q1185775","display_name":"Frame problem","level":2,"score":0.2870999872684479},{"id":"https://openalex.org/C2777868144","wikidata":"https://www.wikidata.org/wiki/Q7239817","display_name":"Preference elicitation","level":3,"score":0.28459998965263367},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.28060001134872437},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.26589998602867126},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.259799987077713},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.257099986076355},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2522999942302704},{"id":"https://openalex.org/C159032336","wikidata":"https://www.wikidata.org/wiki/Q2488768","display_name":"Non-monotonic logic","level":2,"score":0.25110000371932983}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.17890","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.17890","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.17890","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.17890","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Long-form":[0],"chain-of-thought":[1],"reasoning":[2,16,187],"can":[3,71],"improve":[4],"LLM":[5],"performance":[6],"on":[7],"complex":[8],"tasks,":[9],"but":[10],"models":[11],"often":[12],"continue":[13,152],"generating":[14],"unnecessary":[15,111,182],"after":[17,154],"a":[18,46,53,73,90,115,145],"correct":[19],"answer":[20,155],"has":[21],"emerged.":[22],"We":[23,30],"refer":[24],"to":[25,127,189],"this":[26,32,137],"behavior":[27],"as":[28,45],"overthinking.":[29],"study":[31],"phenomenon":[33],"from":[34,109],"the":[35,61,83,106,110,124,159,164,169,173,178,186,191,198],"perspective":[36],"of":[37,63,77,200],"GRPO-style":[38],"reinforcement":[39],"learning":[40],"(RL)":[41],"post-training,":[42],"framing":[43],"it":[44,103],"training-time":[47,146],"credit-assignment":[48],"problem":[49],"rather":[50],"than":[51,79],"merely":[52],"decoding-time":[54],"stopping":[55],"problem.":[56],"In":[57],"rollouts":[58],"sampled":[59],"at":[60],"onset":[62],"GRPO":[64,99],"training,":[65],"we":[66,139],"observe":[67],"that":[68,113,151],"successful":[69,116,149],"trajectories":[70,81,150],"exhibit":[72],"slightly":[74],"higher":[75],"degree":[76],"overthinking":[78,132],"unsuccessful":[80],"for":[82,93,148,181],"same":[84,174],"prompts.":[85],"This":[86],"early":[87],"imbalance":[88,126],"provides":[89],"starting":[91],"point":[92],"an":[94],"undesirable":[95],"feedback":[96],"loop:":[97],"because":[98],"assigns":[100],"sequence-level":[101],"credit,":[102],"cannot":[104],"distinguish":[105],"solution-reaching":[107],"prefix":[108],"continuation":[112],"lengthens":[114],"trajectory.":[117],"Both":[118],"receive":[119],"positive":[120],"update":[121],"signal,":[122],"allowing":[123],"initial":[125],"grow":[128],"into":[129],"more":[130],"severe":[131],"during":[133],"training.":[134],"To":[135],"address":[136],"issue,":[138],"introduce":[140],"Dynamic":[141],"Rollout":[142],"Editing":[143],"(DRE),":[144],"intervention":[147],"thinking":[153,183],"emergence.":[156],"DRE":[157],"preserves":[158],"accepted":[160],"verified":[161],"prefix,":[162],"edits":[163],"remaining":[165],"thinking,":[166],"and":[167],"prefers":[168],"edited":[170],"trajectory":[171],"within":[172],"RL":[175],"group,":[176],"weakening":[177],"preference":[179],"signal":[180],"without":[184],"penalizing":[185],"needed":[188],"reach":[190],"answer.":[192],"Experiments":[193],"across":[194],"diverse":[195],"tasks":[196],"show":[197],"effectiveness":[199],"DRE.":[201]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-18T00:00:00"}