{"id":"https://openalex.org/W7138363444","doi":"https://doi.org/10.1609/aaai.v40i37.40408","title":"Rethinking the Sampling Criteria in Reinforcement Learning for LLM Reasoning: A Competence-Difficulty Alignment Perspective","display_name":"Rethinking the Sampling Criteria in Reinforcement Learning for LLM Reasoning: A Competence-Difficulty Alignment Perspective","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7138363444","doi":"https://doi.org/10.1609/aaai.v40i37.40408"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v40i37.40408","doi":"https://doi.org/10.1609/aaai.v40i37.40408","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i37.40408","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40408/44369","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40408/44369","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125563167","display_name":"Deyang Kong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4405259410","display_name":"Meituan (China)","ror":"https://ror.org/0272kpg92","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405259410"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Deyang Kong","raw_affiliation_strings":["National Engineering Research Center for Software Engineering, Peking University, Beijing, China\nMeituan Group, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Engineering Research Center for Software Engineering, Peking University, Beijing, China\nMeituan Group, Beijing, China","institution_ids":["https://openalex.org/I20231570","https://openalex.org/I4405259410"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5146590990","display_name":"Qi Guo","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4405259410","display_name":"Meituan (China)","ror":"https://ror.org/0272kpg92","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405259410"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qi Guo","raw_affiliation_strings":["National Engineering Research Center for Software Engineering, Peking University, Beijing, China\nMeituan Group, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Engineering Research Center for Software Engineering, Peking University, Beijing, China\nMeituan Group, Beijing, China","institution_ids":["https://openalex.org/I20231570","https://openalex.org/I4405259410"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119753576","display_name":"Xiangyu Xi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405259410","display_name":"Meituan (China)","ror":"https://ror.org/0272kpg92","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405259410"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiangyu Xi","raw_affiliation_strings":["Meituan Group, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meituan Group, Beijing, China","institution_ids":["https://openalex.org/I4405259410"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5146801538","display_name":"Wei Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405259410","display_name":"Meituan (China)","ror":"https://ror.org/0272kpg92","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405259410"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Wang","raw_affiliation_strings":["Meituan Group, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meituan Group, Beijing, China","institution_ids":["https://openalex.org/I4405259410"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139790519","display_name":"Jingang Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405259410","display_name":"Meituan (China)","ror":"https://ror.org/0272kpg92","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405259410"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingang Wang","raw_affiliation_strings":["Meituan Group, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meituan Group, Beijing, China","institution_ids":["https://openalex.org/I4405259410"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068562468","display_name":"Xunliang Cai","orcid":"https://orcid.org/0000-0003-0685-9556","observed_orcids":["https://orcid.org/0000-0003-0685-9556"]},"institutions":[{"id":"https://openalex.org/I4405259410","display_name":"Meituan (China)","ror":"https://ror.org/0272kpg92","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405259410"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xunliang Cai","raw_affiliation_strings":["Meituan Group, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meituan Group, Beijing, China","institution_ids":["https://openalex.org/I4405259410"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101435571","display_name":"Shikun Zhang","orcid":"https://orcid.org/0000-0002-8576-2674","observed_orcids":["https://orcid.org/0000-0002-8576-2674","https://orcid.org/0000-0002-3132-4935"]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shikun Zhang","raw_affiliation_strings":["National Engineering Research Center for Software Engineering, Peking University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Engineering Research Center for Software Engineering, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101538186","display_name":"Wei Ye","orcid":"https://orcid.org/0000-0002-9331-4716","observed_orcids":["https://orcid.org/0000-0002-9331-4716"]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Ye","raw_affiliation_strings":["National Engineering Research Center for Software Engineering, Peking University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Engineering Research Center for Software Engineering, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.12506966,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"37","first_page":"31438","last_page":"31446"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9009000062942505,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9009000062942505,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.019500000402331352,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.012600000016391277,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-reasoning","display_name":"large language model reasoning","score":0.7839999794960022},{"id":"https://openalex.org/keywords/sampling-efficiency","display_name":"sampling efficiency","score":0.6340000033378601},{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.7329999804496765},{"id":"https://openalex.org/keywords/training-efficiency","display_name":"training efficiency","score":0.5019999742507935}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7002999782562256},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6977999806404114},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5914999842643738},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5631999969482422},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.4309000074863434},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.4302999973297119},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3880000114440918},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.38029998540878296},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.33219999074935913},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.3292999863624573},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.31130000948905945},{"id":"https://openalex.org/C52740198","wikidata":"https://www.wikidata.org/wiki/Q1539564","display_name":"Importance sampling","level":3,"score":0.28700000047683716},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.28040000796318054}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1609/aaai.v40i37.40408","doi":"https://doi.org/10.1609/aaai.v40i37.40408","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i37.40408","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40408/44369","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i37.40408","doi":"https://doi.org/10.1609/aaai.v40i37.40408","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i37.40408","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40408/44369","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7138363444.pdf","grobid_xml":"https://content.openalex.org/works/W7138363444.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4417228675","https://openalex.org/W7127428438","https://openalex.org/W4416034889","https://openalex.org/W4415339868","https://openalex.org/W4416374533","https://openalex.org/W4417119453","https://openalex.org/W7162669510","https://openalex.org/W4407759210","https://openalex.org/W4416681990","https://openalex.org/W4415948548"],"abstract_inverted_index":{"The":[0],"low":[1],"sampling":[2],"efficiency":[3,26,153],"during":[4],"the":[5,50,106,128,137,146,152,167],"rollout":[6],"phase":[7],"poses":[8],"a":[9,111],"significant":[10],"challenge":[11],"to":[12,24,48,62,98,145],"scaling":[13],"reinforcement":[14],"learning":[15],"for":[16,78],"large":[17],"language":[18],"model":[19,53,94,161,164],"reasoning.":[20],"Existing":[21],"methods":[22],"attempt":[23],"improve":[25],"by":[27,86,142],"scheduling":[28],"problems":[29,101],"based":[30],"on":[31,158],"problem":[32,44,56,84],"difficulties.":[33],"However,":[34],"these":[35,67],"approaches":[36],"suffer":[37],"from":[38],"unstable":[39],"and":[40,46,55,80,163],"biased":[41],"estimations":[42],"of":[43,83,132,154,170],"difficulty":[45,57],"fail":[47],"capture":[49],"alignment":[51],"between":[52],"competence":[54,95,109],"in":[58,116],"RL":[59,118],"training,":[60],"leading":[61],"suboptimal":[63],"results.":[64],"To":[65],"address":[66],"challenges,":[68],"we":[69],"introduce":[70],"Competence-Difficulty":[71],"Alignment":[72],"Sampling":[73,149],"(CDAS).":[74],"This":[75],"approach":[76],"allows":[77],"accurate":[79],"stable":[81],"estimation":[82],"difficulties":[85,103],"aggregating":[87],"historical":[88],"performance":[89],"discrepancies":[90],"across":[91],"problems.":[92],"Subsequently,":[93],"is":[96],"quantified":[97],"adaptively":[99],"select":[100],"whose":[102],"align":[104],"with":[105],"model's":[107],"current":[108],"using":[110],"fixed-point":[112],"system.":[113],"Extensive":[114],"experiments":[115,157],"mathematical":[117],"training":[119,138],"show":[120],"that":[121],"CDAS":[122,135],"consistently":[123],"outperforms":[124],"strong":[125],"baselines,":[126],"achieving":[127],"highest":[129],"average":[130],"accuracy":[131],"45.89%.":[133],"Furthermore,":[134],"reduces":[136],"step":[139],"time":[140],"overhead":[141],"57.06%":[143],"compared":[144],"widely-used":[147],"Dynamic":[148],"strategy,":[150],"verifying":[151],"CDAS.":[155,171],"Additional":[156],"different":[159],"tasks,":[160],"architectures,":[162],"sizes":[165],"demonstrate":[166],"generalization":[168],"capability":[169]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-02-04T00:00:00"}