{"id":"https://openalex.org/W7163068263","doi":"https://doi.org/10.48550/arxiv.2605.30896","title":"Zero Collapse: A Failure Mode of Policy Gradient Methods in Discontinuous Reward Environments","display_name":"Zero Collapse: A Failure Mode of Policy Gradient Methods in Discontinuous Reward Environments","publication_year":2026,"publication_date":"2026-05-29","ids":{"openalex":"https://openalex.org/W7163068263","doi":"https://doi.org/10.48550/arxiv.2605.30896"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.30896","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.30896","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.30896","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137602903","display_name":"Nishant Kumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kumar, Nishant","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019740318","display_name":"Enrique Areyan Viqueira","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Viqueira, Enrique Areyan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137554116","display_name":"Amy Greenwald","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Greenwald, Amy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9067000150680542,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9067000150680542,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.027799999341368675,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10328","display_name":"Supply Chain and Inventory Management","score":0.012900000438094139,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bidding","display_name":"Bidding","score":0.7245000004768372},{"id":"https://openalex.org/keywords/common-value-auction","display_name":"Common value auction","score":0.6622999906539917},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5679000020027161},{"id":"https://openalex.org/keywords/initialization","display_name":"Initialization","score":0.5257999897003174},{"id":"https://openalex.org/keywords/zero","display_name":"Zero (linguistics)","score":0.5160999894142151},{"id":"https://openalex.org/keywords/mode","display_name":"Mode (computer interface)","score":0.503600001335144},{"id":"https://openalex.org/keywords/overshoot","display_name":"Overshoot (microwave communication)","score":0.47679999470710754},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.33379998803138733}],"concepts":[{"id":"https://openalex.org/C9233905","wikidata":"https://www.wikidata.org/wiki/Q3276328","display_name":"Bidding","level":2,"score":0.7245000004768372},{"id":"https://openalex.org/C163239763","wikidata":"https://www.wikidata.org/wiki/Q5153637","display_name":"Common value auction","level":2,"score":0.6622999906539917},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.618399977684021},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5679000020027161},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.5257999897003174},{"id":"https://openalex.org/C2780813799","wikidata":"https://www.wikidata.org/wiki/Q3274237","display_name":"Zero (linguistics)","level":2,"score":0.5160999894142151},{"id":"https://openalex.org/C48677424","wikidata":"https://www.wikidata.org/wiki/Q6888088","display_name":"Mode (computer interface)","level":2,"score":0.503600001335144},{"id":"https://openalex.org/C2780323453","wikidata":"https://www.wikidata.org/wiki/Q7113957","display_name":"Overshoot (microwave communication)","level":2,"score":0.47679999470710754},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3594000041484833},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.3490999937057495},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.33379998803138733},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.3287999927997589},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.3138999938964844},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.2890999913215637},{"id":"https://openalex.org/C145071142","wikidata":"https://www.wikidata.org/wiki/Q1411116","display_name":"Fictitious play","level":3,"score":0.2870999872684479},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.2849999964237213},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.2703999876976013},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.26919999718666077},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.26649999618530273}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.30896","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.30896","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.30896","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.30896","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Bidding":[0],"in":[1,89],"repeated":[2],"auctions":[3],"is":[4],"a":[5,46,51,59,73,85,156,211],"central":[6],"challenge":[7],"for":[8,30,49,215],"reinforcement":[9],"learning":[10],"(RL),":[11],"combining":[12],"continuous":[13],"control":[14],"with":[15,36],"the":[16,37,64,68,118,131,174],"strategic":[17],"complexities":[18],"of":[19,41,75,120,159,173,187],"digital":[20],"advertising.":[21],"While":[22],"policy":[23,177],"gradient":[24,123],"and":[25,100,111,167,179,182,192,202],"value-based":[26],"methods":[27,137],"seem":[28],"well-suited":[29],"these":[31],"settings,":[32],"they":[33,57],"often":[34],"struggle":[35],"discontinuous,":[38],"\"cliff-like\"":[39],"nature":[40],"auction":[42,216],"reward":[43,55,65],"landscapes.":[44],"In":[45],"first-price":[47],"auction,":[48],"example,":[50],"bidder":[52],"receives":[53],"zero":[54,168],"until":[56],"cross":[58],"specific":[60],"threshold,":[61],"after":[62],"which":[63],"decreases":[66],"as":[67,141],"bid":[69],"increases.":[70],"This":[71],"creates":[72],"landscape":[74],"flat,":[76,113],"zero-reward":[77,114],"regions":[78,110],"separated":[79],"by":[80],"sharp":[81],"boundaries.":[82],"We":[83,95,133,195],"identify":[84],"fundamental":[86],"failure":[87],"mode":[88],"this":[90,147,188],"setting":[91],"termed":[92],"\"zero":[93],"collapse.\"":[94],"show":[96],"that":[97,135],"stochastic":[98],"exploration":[99],"gradient-based":[101],"updates":[102],"can":[103,145],"cause":[104],"policies":[105],"to":[106,164,205],"overshoot":[107],"optimal":[108],"high-reward":[109],"enter":[112],"regimes.":[115],"Once":[116],"there,":[117],"lack":[119],"an":[121,171,184],"informative":[122],"signal":[124],"makes":[125],"recovery":[126],"extremely":[127],"sample-inefficient,":[128],"effectively":[129],"trapping":[130],"agent.":[132],"find":[134],"actor-critic":[136,193],"are":[138],"particularly":[139],"susceptible,":[140],"biased":[142],"value":[143],"estimates":[144],"accelerate":[146],"movement":[148],"toward":[149],"unstable":[150],"regions.":[151],"Our":[152],"contributions":[153],"include:":[154],"(1)":[155],"mechanistic":[157],"explanation":[158],"how":[160],"discontinuous":[161],"rewards":[162],"lead":[163],"vanishing":[165],"signals":[166],"collapse;":[169],"(2)":[170],"analysis":[172],"interaction":[175],"between":[176],"stochasticity":[178],"step":[180],"size;":[181],"(3)":[183],"empirical":[185],"demonstration":[186],"phenomenon":[189],"across":[190],"REINFORCE":[191],"variants.":[194],"propose":[196],"practical":[197],"mitigation":[198],"strategies":[199],"involving":[200],"initialization":[201],"architectural":[203],"choices":[204],"improve":[206],"stability.":[207],"Finally,":[208],"we":[209],"introduce":[210],"formal":[212],"RL":[213],"framework":[214],"environments":[217],"highlighting":[218],"their":[219],"unique":[220],"structural":[221],"properties.":[222]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-02T00:00:00"}