{"id":"https://openalex.org/W7165842766","doi":"https://doi.org/10.48550/arxiv.2606.24994","title":"ExTra: Exploratory Trajectory Optimization for Language Model Reinforcement Learning","display_name":"ExTra: Exploratory Trajectory Optimization for Language Model Reinforcement Learning","publication_year":2026,"publication_date":"2026-06-23","ids":{"openalex":"https://openalex.org/W7165842766","doi":"https://doi.org/10.48550/arxiv.2606.24994"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.24994","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24994","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.24994","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139368003","display_name":"Wenyang Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Wenyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139314790","display_name":"Junxiang Jia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jia, Junxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139333335","display_name":"Zhen Shu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shu, Zhen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137648986","display_name":"Daniel Dahlmeier","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dahlmeier, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139379958","display_name":"See-Kiong Ng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ng, See-Kiong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139383258","display_name":"Bryan Kian Hsiang Low","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Low, Bryan Kian Hsiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2402999997138977,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2402999997138977,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.2328999936580658,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.12080000340938568,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.683899998664856},{"id":"https://openalex.org/keywords/novelty","display_name":"Novelty","score":0.6345000267028809},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.5406000018119812},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.527999997138977},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.41920000314712524},{"id":"https://openalex.org/keywords/interpretability","display_name":"Interpretability","score":0.3824000060558319},{"id":"https://openalex.org/keywords/novelty-detection","display_name":"Novelty detection","score":0.3630000054836273},{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.3255000114440918}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7124999761581421},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.683899998664856},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.671999990940094},{"id":"https://openalex.org/C2778738651","wikidata":"https://www.wikidata.org/wiki/Q16546687","display_name":"Novelty","level":2,"score":0.6345000267028809},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.5406000018119812},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.527999997138977},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5116999745368958},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.41920000314712524},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.3824000060558319},{"id":"https://openalex.org/C2778924833","wikidata":"https://www.wikidata.org/wiki/Q7064603","display_name":"Novelty detection","level":3,"score":0.3630000054836273},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.3255000114440918},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3206999897956848},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.31619998812675476},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.30889999866485596},{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.3027999997138977},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.2705000042915344},{"id":"https://openalex.org/C9679016","wikidata":"https://www.wikidata.org/wiki/Q1417473","display_name":"Principle of maximum entropy","level":2,"score":0.26969999074935913},{"id":"https://openalex.org/C196340769","wikidata":"https://www.wikidata.org/wiki/Q7698910","display_name":"Temporal difference learning","level":3,"score":0.26350000500679016},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.25850000977516174}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.24994","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24994","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.24994","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24994","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"Learning":[1],"with":[2,25,36],"Verifiable":[3],"Rewards":[4],"(RLVR)":[5],"for":[6],"language-model":[7],"reasoning":[8,100],"can":[9,32,123],"fail":[10],"at":[11],"both":[12,125],"extremes":[13],"of":[14],"task":[15],"difficulty:":[16],"easy":[17],"prompts":[18,31],"often":[19],"produce":[20,33],"all-correct,":[21],"low-diversity":[22],"rollout":[23],"groups":[24,35],"little":[26],"gradient":[27],"signal,":[28],"while":[29],"hard":[30],"all-incorrect":[34],"no":[37],"positive":[38],"reward.":[39],"We":[40],"introduce":[41],"ExTra":[42,58,102],"(Exploratory":[43],"Trajectory":[44],"Optimization),":[45],"a":[46,63],"GRPO-compatible":[47],"framework":[48],"that":[49,66,119],"extracts":[50],"exploration":[51,92,121],"signals":[52,89,122],"from":[53,93],"the":[54],"model's":[55],"own":[56],"rollouts.":[57],"combines":[59],"two":[60],"mechanisms:":[61],"(i)":[62],"novelty":[64],"reward":[65],"adds":[67],"embedding-based":[68],"diversity":[69],"bonuses":[70],"after":[71],"GRPO":[72,106],"normalization,":[73],"rewarding":[74],"diverse":[75],"correct":[76],"solutions;":[77],"and":[78,90,113,128],"(ii)":[79],"entropy-guided":[80],"prefix":[81],"regeneration,":[82],"which":[83],"scores":[84],"partial":[85],"trajectories":[86],"using":[87],"entropy":[88],"continues":[91],"promising":[94],"intermediate":[95],"steps.":[96],"Across":[97],"six":[98],"mathematical":[99],"benchmarks,":[101],"improves":[103],"Qwen3-1.7B":[104],"over":[105],"by":[107],"about":[108],"+5":[109],"points":[110,115],"on":[111,116],"pass@1":[112],"+7":[114],"pass@16,":[117],"showing":[118],"trajectory-level":[120],"improve":[124],"single-sample":[126],"accuracy":[127],"inference-time":[129],"coverage.":[130]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-26T00:00:00"}