{"id":"https://openalex.org/W7204201890","doi":"https://doi.org/10.18653/2026.emnlp-main.1586","title":"Beyond the Stability-Exploration Dilemma: Environmental Regularization for LLM Policy Optimization","display_name":"Beyond the Stability-Exploration Dilemma: Environmental Regularization for LLM Policy Optimization","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7204201890","doi":"https://doi.org/10.18653/2026.emnlp-main.1586"},"language":"en","primary_location":{"id":"doi:10.18653/2026.emnlp-main.1586","doi":"https://doi.org/10.18653/2026.emnlp-main.1586","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.1586","pdf_url":"https://aclanthology.org/2026.emnlp-main.1586.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://aclanthology.org/2026.emnlp-main.1586.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113289628","display_name":"Xianlei Zhou","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xianlei Zhou","raw_affiliation_strings":["AMAP , Alibaba Group","Equal contribution"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AMAP , Alibaba Group","institution_ids":["https://openalex.org/I45928872"]},{"raw_affiliation_string":"Equal contribution","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115563621","display_name":"Xiangdi Meng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiangdi Meng","raw_affiliation_strings":["AMAP , Alibaba Group","Equal contribution"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AMAP , Alibaba Group","institution_ids":["https://openalex.org/I45928872"]},{"raw_affiliation_string":"Equal contribution","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147389220","display_name":"Yu He","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu He","raw_affiliation_strings":["AMAP , Alibaba Group","Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AMAP , Alibaba Group","institution_ids":["https://openalex.org/I45928872"]},{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123037483","display_name":"Tianyu Qi","orcid":"https://orcid.org/0000-0001-5484-985X","observed_orcids":["https://orcid.org/0000-0001-5484-985X","https://orcid.org/0009-0006-1616-9902"]},"institutions":[{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianyu Qi","raw_affiliation_strings":["JD.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"JD.com","institution_ids":["https://openalex.org/I4210103986"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082953115","display_name":"Shuyan Guan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuyan Guan","raw_affiliation_strings":["AMAP , Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AMAP , Alibaba Group","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006451453","display_name":"Xianli Zhang","orcid":"https://orcid.org/0000-0003-1995-3904","observed_orcids":["https://orcid.org/0000-0003-1995-3904"]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xianli Zhang","raw_affiliation_strings":["AMAP , Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AMAP , Alibaba Group","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148063861","display_name":"Jian Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I25254941","display_name":"Beijing Normal University","ror":"https://ror.org/022k4wk35","country_code":"CN","type":"education","lineage":["https://openalex.org/I25254941"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian Zhang","raw_affiliation_strings":["Beijing Normal University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing Normal University","institution_ids":["https://openalex.org/I25254941"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148089677","display_name":"Xin Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Li","raw_affiliation_strings":["AMAP , Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AMAP , Alibaba Group","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133574086","display_name":"Qika Lin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Qika Lin","raw_affiliation_strings":["National University of Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5133555150","display_name":"Jun Liu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Liu","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.55742439,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"36071","last_page":"36086"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.7509999871253967,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.7509999871253967,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.06859999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.03629999980330467,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8339999914169312},{"id":"https://openalex.org/keywords/policy-optimization","display_name":"policy optimization","score":0.9020000100135803},{"id":"https://openalex.org/keywords/kl-regularization","display_name":"KL regularization","score":0.6380000114440918},{"id":"https://openalex.org/keywords/environmental-regularization","display_name":"environmental regularization","score":0.6809999942779541},{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.7429999709129333},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"reinforcement learning","score":0.4180000126361847}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.66839998960495},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.5375000238418579},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.512499988079071},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.4772999882698059},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.42899999022483826},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.38019999861717224},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.3741999864578247},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3668999969959259},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.3637000024318695},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.35839998722076416},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.32829999923706055},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.30570000410079956},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.3043000102043152},{"id":"https://openalex.org/C152442038","wikidata":"https://www.wikidata.org/wiki/Q2778212","display_name":"Tikhonov regularization","level":3,"score":0.29089999198913574},{"id":"https://openalex.org/C2778496695","wikidata":"https://www.wikidata.org/wiki/Q254128","display_name":"Dilemma","level":2,"score":0.2896000146865845},{"id":"https://openalex.org/C115680565","wikidata":"https://www.wikidata.org/wiki/Q5977448","display_name":"Gradient method","level":2,"score":0.2840999960899353},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.2727999985218048},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.27070000767707825},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.2678000032901764},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2526000142097473}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/2026.emnlp-main.1586","doi":"https://doi.org/10.18653/2026.emnlp-main.1586","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.1586","pdf_url":"https://aclanthology.org/2026.emnlp-main.1586.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2608.23311","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.23311","pdf_url":"https://arxiv.org/pdf/2608.23311","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.23311","doi":"https://doi.org/10.48550/arxiv.2608.23311","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.23311","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/2026.emnlp-main.1586","doi":"https://doi.org/10.18653/2026.emnlp-main.1586","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.1586","pdf_url":"https://aclanthology.org/2026.emnlp-main.1586.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204201890.pdf","grobid_xml":"https://content.openalex.org/works/W7204201890.grobid-xml"},"referenced_works_count":16,"referenced_works":["https://openalex.org/W112666333","https://openalex.org/W2736601468","https://openalex.org/W4223908421","https://openalex.org/W4310290453","https://openalex.org/W4391631327","https://openalex.org/W4402671952","https://openalex.org/W4402672014","https://openalex.org/W4403746689","https://openalex.org/W4403788929","https://openalex.org/W4404344587","https://openalex.org/W4404352912","https://openalex.org/W4405655184","https://openalex.org/W4409049124","https://openalex.org/W4416070632","https://openalex.org/W6891896986","https://openalex.org/W7196953391"],"related_works":["https://openalex.org/W7125207993","https://openalex.org/W4414979505","https://openalex.org/W7221015659","https://openalex.org/W7128018134","https://openalex.org/W7160905086","https://openalex.org/W4416774044","https://openalex.org/W4416245727","https://openalex.org/W7119492051","https://openalex.org/W7140090605","https://openalex.org/W7164034211"],"abstract_inverted_index":{"Policy":[0,81],"optimization":[1,40],"(PO)":[2],"for":[3,47],"Large":[4],"Language":[5],"Models":[6],"faces":[7],"a":[8,21,85,97],"stability-exploration":[9],"trade-off,":[10],"currently":[11],"mediated":[12],"by":[13,54,69,126],"an":[14,42,48],"action-side":[15,32],"Policy-KL":[16,25,165],"regularizer.This":[17],"puts":[18],"practitioners":[19],"in":[20,132],"double":[22],"bind:":[23],"keeping":[24],"constrains":[26],"response":[27,122,145],"behavior":[28,182],"and":[29,178,186],"consumes":[30],"the":[31,39,52,58,63,70,111,118,121,133,144,163],"exploration":[33],"budget,":[34],"while":[35,167],"dropping":[36],"it":[37],"leaves":[38],"without":[41,153],"explicit":[43],"drift":[44],"control.We":[45],"argue":[46],"alternative":[49],"that":[50,89,102],"breaks":[51],"dilemma":[53],"moving":[55],"regularization":[56],"to":[57],"input":[59],"side.As":[60],"training":[61,66],"progresses,":[62],"distribution":[64,93,173],"over":[65,171],"queries":[67,108],"induced":[68],"current":[71],"policy":[72],"drifts":[73],"unchecked":[74],"from":[75],"its":[76],"pre-RL":[77],"reference":[78],"distribution.Concretely,":[79],"Environment-Regularized":[80],"Optimization":[82],"(ERPO)":[83],"introduces":[84],"Query-KL":[86],"(QKL)":[87],"term":[88],"bounds":[90],"this":[91],"query":[92,119,172],"shift,":[94],"together":[95],"with":[96],"dataset-static":[98],"reference-derived":[99],"perquery":[100],"weight":[101],"biases":[103],"each":[104],"per-query":[105],"update":[106],"toward":[107],"typical":[109],"under":[110,183],"reference.The":[112],"QKL":[113,134,137],"gradient":[114,141],"flows":[115],"strictly":[116],"through":[117],"likelihood;":[120],"score":[123],"function":[124],"used":[125],"policy-gradient":[127],"estimators":[128],"does":[129],"not":[130],"appear":[131],"term,":[135],"so":[136],"exerts":[138],"no":[139],"direct":[140],"pressure":[142],"on":[143],"distributionexploration":[146],"is":[147],"preserved.ERPO":[148],"plugs":[149],"into":[150],"GRPO/PPO/REINFORCE-style":[151],"pipelines":[152],"additional":[154],"forward":[155],"passes.On":[156],"six":[157],"mathematical":[158],"reasoning":[159],"benchmarks,":[160],"ERPO":[161],"replaces":[162],"standard":[164],"regularizer":[166],"achieving":[168],"effective":[169],"control":[170],"drift,":[174],"delivering":[175],"stronger":[176],"accuracy":[177],"substantially":[179],"more":[180],"stable":[181],"high-temperature":[184],"decoding":[185],"long-horizon":[187],"training.":[188]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-26T00:00:00"}