{"id":"https://openalex.org/W7155059120","doi":"https://doi.org/10.48550/arxiv.2604.16918","title":"Freshness-Aware Prioritized Experience Replay for LLM/VLM Reinforcement Learning","display_name":"Freshness-Aware Prioritized Experience Replay for LLM/VLM Reinforcement Learning","publication_year":2026,"publication_date":"2026-04-18","ids":{"openalex":"https://openalex.org/W7155059120","doi":"https://doi.org/10.48550/arxiv.2604.16918"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.16918","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.16918","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.16918","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100510724","display_name":"Weiyu Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Weiyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035793423","display_name":"Yongcheng Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Yongcheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134151006","display_name":"Yan Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Yan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134144679","display_name":"Xinyu Cui","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cui, Xinyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134127069","display_name":"Jian Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Jian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134204369","display_name":"Xuhui Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xuhui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134127928","display_name":"Mohamed Elhoseiny","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Elhoseiny, Mohamed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.6499000191688538,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.6499000191688538,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.05260000005364418,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.03959999978542328,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.8025000095367432},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.6744999885559082},{"id":"https://openalex.org/keywords/reuse","display_name":"Reuse","score":0.48179998993873596},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.47690001130104065},{"id":"https://openalex.org/keywords/multiplicative-function","display_name":"Multiplicative function","score":0.426800012588501},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.3736000061035156},{"id":"https://openalex.org/keywords/code-reuse","display_name":"Code reuse","score":0.335999995470047}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8025000095367432},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6883999705314636},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.6744999885559082},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.48179998993873596},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.47690001130104065},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4388999938964844},{"id":"https://openalex.org/C42747912","wikidata":"https://www.wikidata.org/wiki/Q1048447","display_name":"Multiplicative function","level":2,"score":0.426800012588501},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4023999869823456},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3736000061035156},{"id":"https://openalex.org/C2778583558","wikidata":"https://www.wikidata.org/wiki/Q771245","display_name":"Code reuse","level":3,"score":0.335999995470047},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.335099995136261},{"id":"https://openalex.org/C91306197","wikidata":"https://www.wikidata.org/wiki/Q45767","display_name":"Competition (biology)","level":2,"score":0.3093000054359436},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.304500013589859},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.28529998660087585},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.274399995803833},{"id":"https://openalex.org/C149629883","wikidata":"https://www.wikidata.org/wiki/Q660926","display_name":"Fraction (chemistry)","level":2,"score":0.26989999413490295},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.25940001010894775}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.16918","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.16918","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.16918","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.16918","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"Learning":[1],"(RL)":[2],"has":[3],"achieved":[4],"impressive":[5],"success":[6],"in":[7,44,65,134],"post-training":[8],"Large":[9],"Language":[10],"Models":[11,15],"(LLMs)":[12],"and":[13,24,75,166,173,190],"Vision-Language":[14],"(VLMs),":[16],"with":[17,127,170],"on-policy":[18,180],"algorithms":[19],"such":[20],"as":[21,27],"PPO,":[22],"GRPO,":[23],"REINFORCE++":[25],"serving":[26],"the":[28,140,148],"dominant":[29],"paradigm.":[30],"However,":[31],"these":[32],"methods":[33],"discard":[34],"all":[35],"collected":[36],"trajectories":[37,74,102],"after":[38,107],"a":[39,128],"single":[40],"gradient":[41],"update,":[42],"resulting":[43],"poor":[45],"sample":[46,63,136],"efficiency,":[47],"particularly":[48],"wasteful":[49],"for":[50],"agentic":[51],"tasks":[52,169],"where":[53],"multi-turn":[54],"environment":[55],"interactions":[56],"are":[57],"expensive.":[58],"While":[59],"Experience":[60,82],"Replay":[61,83],"drives":[62],"efficiency":[64],"classic":[66],"RL":[67],"by":[68,122],"allowing":[69],"agents":[70],"to":[71,85,103,151,155],"reuse":[72],"past":[73],"prioritize":[76],"informative":[77],"ones,":[78],"directly":[79],"applying":[80],"Prioritized":[81],"(PER)":[84],"LLMs":[86],"fails.":[87],"The":[88],"rapid":[89],"policy":[90],"evolution":[91],"of":[92,142],"billion-parameter":[93],"models":[94],"renders":[95],"stored":[96],"priorities":[97],"stale,":[98],"causing":[99],"old":[100],"high-priority":[101],"dominate":[104],"sampling":[105],"long":[106],"they":[108],"have":[109],"become":[110],"uninformative.":[111],"We":[112,159],"propose":[113],"Freshness-Aware":[114,145,176],"PER,":[115],"which":[116],"addresses":[117],"this":[118],"priority":[119,126],"staleness":[120],"problem":[121],"augmenting":[123],"any":[124],"PER-based":[125],"multiplicative":[129],"exponential":[130],"age":[131,199],"decay":[132,200],"grounded":[133],"effective":[135],"size":[137],"analysis.":[138],"To":[139],"best":[141],"our":[143],"knowledge,":[144],"PER":[146,154,177,197],"is":[147,206],"first":[149],"work":[150],"successfully":[152],"apply":[153],"LLM/VLM":[156],"reinforcement":[157],"learning.":[158],"evaluate":[160],"on":[161,184,188,192],"eight":[162],"multi-step":[163],"agentic,":[164],"reasoning,":[165],"math":[167],"competition":[168],"0.5B,":[171],"3B,":[172],"7B":[174],"models.":[175],"significantly":[178],"outperforms":[179],"baselines,":[181],"achieving":[182],"+46%":[183],"NQ":[185],"Search,":[186],"+367%":[187],"Sokoban,":[189],"+133%":[191],"VLM":[193],"FrozenLake,":[194],"while":[195],"standard":[196],"without":[198],"consistently":[201],"degrades":[202],"performance.":[203],"Our":[204],"code":[205],"publicly":[207],"available":[208],"at":[209],"https://github.com/Vision-CAIR/Freshness-Aware-PER.":[210]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-22T00:00:00"}