{"id":"https://openalex.org/W7130728993","doi":"https://doi.org/10.48550/arxiv.2602.17497","title":"Retrospective In-Context Learning for Temporal Credit Assignment with Large Language Models","display_name":"Retrospective In-Context Learning for Temporal Credit Assignment with Large Language Models","publication_year":2026,"publication_date":"2026-02-19","ids":{"openalex":"https://openalex.org/W7130728993","doi":"https://doi.org/10.48550/arxiv.2602.17497"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2602.17497","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2602.17497","pdf_url":"https://arxiv.org/pdf/2602.17497","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2602.17497","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113084765","display_name":"Wentse Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chen, Wen-Tse","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126482811","display_name":"Jiayu Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Chen, Jiayu","raw_affiliation_strings":["The University of Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046483574","display_name":"Fahim Tajwar","orcid":"https://orcid.org/0000-0001-9257-6282","observed_orcids":["https://orcid.org/0000-0001-9257-6282"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tajwar, Fahim","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126461266","display_name":"Hao Zhu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhu, Hao","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126448043","display_name":"Xintong Duan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Duan, Xintong","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071983998","display_name":"Ruslan Salakhutdinov","orcid":"https://orcid.org/0000-0002-3752-2756","observed_orcids":["https://orcid.org/0000-0002-3752-2756"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Salakhutdinov, Ruslan","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5127850841","display_name":"Jeff Schneider","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Schneider, Jeff","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/temporal-credit-assignment","display_name":"temporal credit assignment","score":0.984000027179718},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.968999981880188},{"id":"https://openalex.org/keywords/sparse-rewards","display_name":"sparse rewards","score":0.5659999847412109},{"id":"https://openalex.org/keywords/online-reinforcement-learning","display_name":"online reinforcement learning","score":0.7269999980926514},{"id":"https://openalex.org/keywords/sample-efficiency","display_name":"sample efficiency","score":0.9049999713897705},{"id":"https://openalex.org/keywords/self-evolving-agents","display_name":"self-evolving agents","score":0.7480000257492065}],"concepts":[{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.8709999918937683},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6978999972343445},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5914999842643738},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5748000144958496},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.531499981880188},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4318999946117401},{"id":"https://openalex.org/C178350159","wikidata":"https://www.wikidata.org/wiki/Q162714","display_name":"Credit risk","level":2,"score":0.39660000801086426},{"id":"https://openalex.org/C196340769","wikidata":"https://www.wikidata.org/wiki/Q7698910","display_name":"Temporal difference learning","level":3,"score":0.38960000872612},{"id":"https://openalex.org/C58973888","wikidata":"https://www.wikidata.org/wiki/Q1041418","display_name":"Semi-supervised learning","level":2,"score":0.32330000400543213},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3165000081062317},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.28139999508857727},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.2808000147342682},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.2777999937534332},{"id":"https://openalex.org/C2986087404","wikidata":"https://www.wikidata.org/wiki/Q15946010","display_name":"Online learning","level":2,"score":0.26010000705718994},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.25859999656677246},{"id":"https://openalex.org/C24138899","wikidata":"https://www.wikidata.org/wiki/Q17141258","display_name":"Instance-based learning","level":3,"score":0.2535000145435333}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2602.17497","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2602.17497","pdf_url":"https://arxiv.org/pdf/2602.17497","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2602.17497","doi":"https://doi.org/10.48550/arxiv.2602.17497","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.17497","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2602.17497","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2602.17497","pdf_url":"https://arxiv.org/pdf/2602.17497","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1912064254","display_name":"Track 4:  Advanced CI Coordination Ecosystem: Monitoring and Measurement Services","funder_award_id":"2137603","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4006316252","display_name":"Track 1: ACCESS Resource Allocations Marketplace and Platform Services (RAMPS)","funder_award_id":"2138259","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4676224565","display_name":"ACO: An Open CI Ecosystem to Advance Scientific Discovery (OpenCI)","funder_award_id":"2138296","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G509285653","display_name":"Track 3: COre National Ecosystem for CyberinfrasTructure (CONECT)","funder_award_id":"2138307","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8300691027","display_name":"Track 2: Customized Multi-tier Assistance, Training, and Computational Help (MATCH) for End User ACCESS to CI","funder_award_id":"2138286","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7130728993.pdf","grobid_xml":"https://content.openalex.org/works/W7130728993.grobid-xml"},"referenced_works_count":23,"referenced_works":["https://openalex.org/W1191599655","https://openalex.org/W1771410628","https://openalex.org/W2109169869","https://openalex.org/W2785738552","https://openalex.org/W2907923173","https://openalex.org/W2963577640","https://openalex.org/W2978455699","https://openalex.org/W4322825501","https://openalex.org/W4380993732","https://openalex.org/W4383472635","https://openalex.org/W4385644214","https://openalex.org/W4387355948","https://openalex.org/W4387839203","https://openalex.org/W4402670760","https://openalex.org/W4402672014","https://openalex.org/W4403747985","https://openalex.org/W4404649840","https://openalex.org/W4404783228","https://openalex.org/W4411063470","https://openalex.org/W4415795685","https://openalex.org/W4417167303","https://openalex.org/W6929276109","https://openalex.org/W7113260674"],"related_works":["https://openalex.org/W7223147483","https://openalex.org/W4403747985","https://openalex.org/W7138444981","https://openalex.org/W7221812116","https://openalex.org/W4409363323","https://openalex.org/W7203761647","https://openalex.org/W7027420228","https://openalex.org/W7080979222","https://openalex.org/W7212730484","https://openalex.org/W7202159264"],"abstract_inverted_index":{"Learning":[0],"from":[1,45,61,102],"self-sampled":[2],"data":[3],"and":[4,49,118,171],"sparse":[5,24,68],"environmental":[6],"feedback":[7,25],"remains":[8],"a":[9],"fundamental":[10],"challenge":[11],"in":[12,123],"training":[13,72],"self-evolving":[14],"agents.":[15],"Temporal":[16],"credit":[17,41,99,128,163],"assignment":[18,100],"mitigates":[19],"this":[20,53],"issue":[21],"by":[22],"transforming":[23],"into":[26,70],"dense":[27,71],"supervision":[28],"signals.":[29],"However,":[30],"previous":[31],"approaches":[32],"typically":[33],"depend":[34],"on":[35,97,132],"learning":[36,81,88],"task-specific":[37],"value":[38],"functions":[39],"for":[40,126,161,168],"assignment,":[42,164],"which":[43,91],"suffer":[44],"poor":[46],"sample":[47,151],"efficiency":[48],"limited":[50,116],"generalization.":[51],"In":[52],"work,":[54],"we":[55],"propose":[56,85],"to":[57,66],"leverage":[58],"pretrained":[59],"knowledge":[60],"large":[62],"language":[63],"models":[64],"(LLMs)":[65],"transform":[67],"rewards":[69],"signals":[73],"(i.e.,":[74],"the":[75,94,98,112,124,156,166],"advantage":[76,113],"function)":[77],"through":[78],"retrospective":[79],"in-context":[80],"(RICL).":[82],"We":[83,104],"further":[84],"an":[86],"online":[87,145],"framework,":[89],"RICOL,":[90],"iteratively":[92],"refines":[93],"policy":[95],"based":[96],"results":[101],"RICL.":[103],"empirically":[105],"demonstrate":[106],"that":[107,137],"RICL":[108],"can":[109],"accurately":[110],"estimate":[111],"function":[114],"with":[115,143,148],"samples":[117],"effectively":[119],"identify":[120],"critical":[121],"states":[122],"environment":[125],"temporal":[127,162],"assignment.":[129],"Extended":[130],"evaluation":[131],"four":[133],"BabyAI":[134],"scenarios":[135],"show":[136],"RICOL":[138],"achieves":[139],"comparable":[140],"convergent":[141],"performance":[142],"traditional":[144],"RL":[146,173],"algorithms":[147],"significantly":[149],"higher":[150],"efficiency.":[152],"Our":[153],"findings":[154],"highlight":[155],"potential":[157],"of":[158],"leveraging":[159],"LLMs":[160],"paving":[165],"way":[167],"more":[169],"sample-efficient":[170],"generalizable":[172],"paradigms.":[174]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-02-21T00:00:00"}