{"id":"https://openalex.org/W4283022083","doi":"https://doi.org/10.48550/arxiv.2206.06827","title":"Variance Reduction for Policy-Gradient Methods via Empirical Variance Minimization","display_name":"Variance Reduction for Policy-Gradient Methods via Empirical Variance Minimization","publication_year":2022,"publication_date":"2022-06-14","ids":{"openalex":"https://openalex.org/W4283022083","doi":"https://doi.org/10.48550/arxiv.2206.06827"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2206.06827","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2206.06827","pdf_url":"https://arxiv.org/pdf/2206.06827","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2206.06827","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054772684","display_name":"Maxim Kaledin","orcid":"https://orcid.org/0000-0002-0925-3130","observed_orcids":["https://orcid.org/0000-0002-0925-3130"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kaledin, Maxim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Golubev, Alexander","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I118501908","display_name":"National Research University Higher School of Economics","ror":"https://ror.org/055f7t516","country_code":"RU","type":"education","lineage":["https://openalex.org/I118501908"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Golubev, Alexander","raw_affiliation_strings":["HDI Lab , HSE University , Moscow , Russia;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HDI Lab , HSE University , Moscow , Russia;","institution_ids":["https://openalex.org/I118501908"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023650696","display_name":"Denis Vital'evich Belomestny","orcid":"https://orcid.org/0000-0002-9482-6430","observed_orcids":["https://orcid.org/0000-0002-9482-6430"]},"institutions":[{"id":"https://openalex.org/I118501908","display_name":"National Research University Higher School of Economics","ror":"https://ror.org/055f7t516","country_code":"RU","type":"education","lineage":["https://openalex.org/I118501908"]},{"id":"https://openalex.org/I62318514","display_name":"University of Duisburg-Essen","ror":"https://ror.org/04mz5ra38","country_code":"DE","type":"education","lineage":["https://openalex.org/I62318514"]}],"countries":["DE","RU"],"is_corresponding":false,"raw_author_name":"Belomestny, Denis","raw_affiliation_strings":["HSE University , Moscow , Russia;","University of Duisburg-Essen , Essen , Germany; HDI Lab ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HSE University , Moscow , Russia;","institution_ids":["https://openalex.org/I118501908"]},{"raw_affiliation_string":"University of Duisburg-Essen , Essen , Germany; HDI Lab ,","institution_ids":["https://openalex.org/I62318514"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11195","display_name":"Simulation Techniques and Applications","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/policy-gradient","display_name":"policy gradient","score":0.722000002861023},{"id":"https://openalex.org/keywords/variance-reduction","display_name":"variance reduction","score":0.9800000190734863},{"id":"https://openalex.org/keywords/control-variates","display_name":"control variates","score":0.925000011920929},{"id":"https://openalex.org/keywords/a2c","display_name":"Advantage Actor-Critic","score":0.9810000061988831},{"id":"https://openalex.org/keywords/actor-critic","display_name":"actor-critic","score":0.7950000166893005},{"id":"https://openalex.org/keywords/deep-reinforcement-learning","display_name":"deep reinforcement learning","score":0.8519999980926514},{"id":"https://openalex.org/keywords/theoretical-guarantees","display_name":"theoretical guarantees","score":0.7509999871253967}],"concepts":[{"id":"https://openalex.org/C62644790","wikidata":"https://www.wikidata.org/wiki/Q3454689","display_name":"Variance reduction","level":3,"score":0.7898119688034058},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.7752625346183777},{"id":"https://openalex.org/C121683094","wikidata":"https://www.wikidata.org/wiki/Q3554721","display_name":"Control variates","level":5,"score":0.7162235975265503},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.599836528301239},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.5896413326263428},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5432594418525696},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5005710124969482},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.45954886078834534},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.4305270314216614},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.35714179277420044},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3282274007797241},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.32680854201316833},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.321216881275177},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.19666028022766113},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.08647233247756958},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.0},{"id":"https://openalex.org/C111350023","wikidata":"https://www.wikidata.org/wiki/Q1191869","display_name":"Markov chain Monte Carlo","level":3,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C13153151","wikidata":"https://www.wikidata.org/wiki/Q1639846","display_name":"Hybrid Monte Carlo","level":4,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2206.06827","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2206.06827","pdf_url":"https://arxiv.org/pdf/2206.06827","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2206.06827","doi":"https://doi.org/10.48550/arxiv.2206.06827","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2206.06827","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2206.06827","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2206.06827","pdf_url":"https://arxiv.org/pdf/2206.06827","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320324261","display_name":"National Research University Higher School of Economics","ror":"https://ror.org/055f7t516"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4283022083.pdf","grobid_xml":"https://content.openalex.org/works/W4283022083.grobid-xml"},"referenced_works_count":11,"referenced_works":["https://openalex.org/W1632550705","https://openalex.org/W2119717200","https://openalex.org/W2766447205","https://openalex.org/W2981030070","https://openalex.org/W2982316857","https://openalex.org/W2989931031","https://openalex.org/W3037207827","https://openalex.org/W3084196880","https://openalex.org/W3100366369","https://openalex.org/W3109546547","https://openalex.org/W4245577611"],"related_works":["https://openalex.org/W2962999904","https://openalex.org/W2967027099","https://openalex.org/W7099576460","https://openalex.org/W2901418935","https://openalex.org/W2952191563","https://openalex.org/W2108682071","https://openalex.org/W2786887383","https://openalex.org/W2112239212","https://openalex.org/W2135290313","https://openalex.org/W2766436268"],"abstract_inverted_index":{"Policy-gradient":[0],"methods":[1,76,188],"in":[2,11,62,128,182],"Reinforcement":[3],"Learning(RL)":[4],"are":[5,189],"very":[6,161],"universal":[7],"and":[8,34,56,164,194],"widely":[9],"applied":[10],"practice":[12],"but":[13,140],"their":[14],"performance":[15,119],"suffers":[16],"from":[17,147],"the":[18,22,39,47,63,78,82,85,87,92,101,105,114,118,121,129,156],"high":[19],"variance":[20,158,185,197],"of":[21,49,65,74,84,95,120,155,184],"gradient":[23],"estimate.":[24],"Several":[25],"procedures":[26,59],"were":[27],"proposed":[28],"to":[29,46],"reduce":[30],"it":[31],"including":[32],"actor-critic(AC)":[33],"advantage":[35],"actor-critic(A2C)":[36],"methods.":[37],"Recently":[38],"approaches":[40],"have":[41],"got":[42],"new":[43,53,57],"perspective":[44],"due":[45],"introduction":[48],"Deep":[50],"RL:":[51],"both":[52],"control":[54],"variates(CV)":[55],"sub-sampling":[58],"became":[60],"available":[61],"setting":[64],"complex":[66],"models":[67],"like":[68],"neural":[69],"networks.":[70],"The":[71],"vital":[72],"part":[73],"CV-based":[75],"is":[77,91,103,171],"goal":[79,169],"functional":[80,170],"for":[81,113,175],"training":[83],"CV,":[86],"most":[88],"popular":[89],"one":[90,107,122],"least-squares":[93,168],"criterion":[94,102],"A2C.":[96],"Despite":[97],"its":[98],"practical":[99],"success,":[100],"not":[104,132,136],"only":[106,133],"possible.":[108],"In":[109],"this":[110],"paper":[111],"we":[112,149],"first":[115],"time":[116],"investigate":[117],"called":[123],"Empirical":[124],"Variance(EV).":[125],"We":[126],"observe":[127],"experiments":[130,179],"that":[131,166,181],"EV-criterion":[134],"performs":[135],"worse":[137],"than":[138,192],"A2C":[139,167,193],"sometimes":[141],"can":[142],"be":[143],"considerably":[144],"better.":[145],"Apart":[146],"that,":[148],"also":[150],"prove":[151],"some":[152],"theoretical":[153],"guarantees":[154],"actual":[157],"reduction":[159,186],"under":[160],"general":[162],"assumptions":[163],"show":[165],"an":[172],"upper":[173],"bound":[174],"EV":[176],"goal.":[177],"Our":[178],"indicate":[180],"terms":[183],"EV-based":[187],"much":[190],"better":[191],"allow":[195],"stronger":[196],"reduction.":[198]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":4}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2022-06-18T00:00:00"}
