{"id":"https://openalex.org/W7151536958","doi":"https://doi.org/10.48550/arxiv.2604.04686","title":"On the \"Causality\" Step in Policy Gradient Derivations: A Pedagogical Reconciliation of Full Return and Reward-to-Go","display_name":"On the \"Causality\" Step in Policy Gradient Derivations: A Pedagogical Reconciliation of Full Return and Reward-to-Go","publication_year":2026,"publication_date":"2026-04-06","ids":{"openalex":"https://openalex.org/W7151536958","doi":"https://doi.org/10.48550/arxiv.2604.04686"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.04686","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.04686","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.04686","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091554978","display_name":"Nima H. Siboni","orcid":"https://orcid.org/0000-0002-0305-5870"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Siboni, Nima H.","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5091554978"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.4221999943256378,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.4221999943256378,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12210","display_name":"Policy Transfer and Learning","score":0.12150000035762787,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13825","display_name":"Economic Policies and Impacts","score":0.053599998354911804,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/prefix","display_name":"Prefix","score":0.5957000255584717},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.5759999752044678},{"id":"https://openalex.org/keywords/corollary","display_name":"Corollary","score":0.5285999774932861},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.5065000057220459},{"id":"https://openalex.org/keywords/argument","display_name":"Argument (complex analysis)","score":0.5012000203132629},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.4966000020503998},{"id":"https://openalex.org/keywords/statement","display_name":"Statement (logic)","score":0.45010000467300415},{"id":"https://openalex.org/keywords/causality","display_name":"Causality (physics)","score":0.3677999973297119}],"concepts":[{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.5957000255584717},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.5759999752044678},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.5741999745368958},{"id":"https://openalex.org/C2780012671","wikidata":"https://www.wikidata.org/wiki/Q1343870","display_name":"Corollary","level":2,"score":0.5285999774932861},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.5065000057220459},{"id":"https://openalex.org/C98184364","wikidata":"https://www.wikidata.org/wiki/Q1780131","display_name":"Argument (complex analysis)","level":2,"score":0.5012000203132629},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.4966000020503998},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.46939998865127563},{"id":"https://openalex.org/C2777026412","wikidata":"https://www.wikidata.org/wiki/Q2684591","display_name":"Statement (logic)","level":2,"score":0.45010000467300415},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.4253000020980835},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.40220001339912415},{"id":"https://openalex.org/C64357122","wikidata":"https://www.wikidata.org/wiki/Q1149766","display_name":"Causality (physics)","level":2,"score":0.3677999973297119},{"id":"https://openalex.org/C2778701210","wikidata":"https://www.wikidata.org/wiki/Q28130034","display_name":"Constructive","level":3,"score":0.3605000078678131},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.35190001130104065},{"id":"https://openalex.org/C108710211","wikidata":"https://www.wikidata.org/wiki/Q11538","display_name":"Mathematical proof","level":2,"score":0.3165999948978424},{"id":"https://openalex.org/C2777686260","wikidata":"https://www.wikidata.org/wiki/Q144037","display_name":"Calculus (dental)","level":2,"score":0.31529998779296875},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.310699999332428},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.3061999976634979},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.2797999978065491},{"id":"https://openalex.org/C11534374","wikidata":"https://www.wikidata.org/wiki/Q7805280","display_name":"Time consistency","level":2,"score":0.27219998836517334},{"id":"https://openalex.org/C2778023277","wikidata":"https://www.wikidata.org/wiki/Q321703","display_name":"Premise","level":2,"score":0.26440000534057617},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.2597000002861023},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.25690001249313354},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.25429999828338623}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.04686","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.04686","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.04686","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.04686","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.6501781940460205,"display_name":"Reduced inequalities"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"In":[0,114],"introductory":[1],"presentations":[2],"of":[3,44,88,126],"policy":[4],"gradients,":[5],"one":[6],"often":[7],"derives":[8],"the":[9,13,23,30,50,72,81,107,117,127],"REINFORCE":[10],"estimator":[11],"using":[12],"full":[14,24,98],"trajectory":[15,69],"return":[16,25],"and":[17,60,71],"then":[18],"states,":[19],"by":[20,29],"``causality,''":[21],"that":[22,46,58,102],"may":[26],"be":[27],"replaced":[28],"reward-to-go.":[31],"Although":[32],"this":[33,115],"statement":[34],"is":[35,38,85,109,121],"correct,":[36],"it":[37,100],"frequently":[39],"presented":[40],"at":[41],"a":[42,62,92,124],"level":[43],"rigor":[45],"leaves":[47],"unclear":[48],"where":[49],"past-reward":[51],"terms":[52],"disappear.":[53],"This":[54],"short":[55],"paper":[56],"isolates":[57],"step":[59],"gives":[61],"mathematically":[63],"explicit":[64],"derivation":[65,128],"based":[66],"on":[67],"prefix":[68,112],"distributions":[70],"score-function":[73],"identity.":[74],"The":[75],"resulting":[76],"account":[77],"does":[78],"not":[79],"change":[80],"estimator.":[82],"Its":[83],"contribution":[84],"conceptual:":[86],"instead":[87],"presenting":[89],"reward-to-go":[90,103],"as":[91,123,131],"post":[93],"hoc":[94],"unbiased":[95],"replacement":[96],"for":[97],"return,":[99],"shows":[101],"arises":[104],"directly":[105],"once":[106],"objective":[108],"decomposed":[110],"over":[111],"trajectories.":[113],"formulation,":[116],"usual":[118],"causality":[119],"argument":[120],"recovered":[122],"corollary":[125],"rather":[129],"than":[130],"an":[132],"additional":[133],"heuristic":[134],"principle.":[135]},"counts_by_year":[],"updated_date":"2026-04-08T06:07:18.267832","created_date":"2026-04-08T00:00:00"}