{"id":"https://openalex.org/W7161068902","doi":"https://doi.org/10.48550/arxiv.2605.12410","title":"Model-based Bootstrap of Controlled Markov Chains","display_name":"Model-based Bootstrap of Controlled Markov Chains","publication_year":2026,"publication_date":"2026-05-12","ids":{"openalex":"https://openalex.org/W7161068902","doi":"https://doi.org/10.48550/arxiv.2605.12410"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.12410","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12410","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.12410","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136065675","display_name":"Ziwei Su","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Su, Ziwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136067943","display_name":"Imon Banerjee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Banerjee, Imon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136029278","display_name":"Diego Klabjan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Klabjan, Diego","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9379000067710876,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9379000067710876,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.00800000037997961,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.004100000020116568,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.5979999899864197},{"id":"https://openalex.org/keywords/markov-chain","display_name":"Markov chain","score":0.573199987411499},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.49160000681877136},{"id":"https://openalex.org/keywords/central-limit-theorem","display_name":"Central limit theorem","score":0.482699990272522},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.40639999508857727},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.38199999928474426},{"id":"https://openalex.org/keywords/differentiable-function","display_name":"Differentiable function","score":0.3578000068664551},{"id":"https://openalex.org/keywords/limit","display_name":"Limit (mathematics)","score":0.35440000891685486}],"concepts":[{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.6414999961853027},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.5979999899864197},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.573199987411499},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.49160000681877136},{"id":"https://openalex.org/C166785042","wikidata":"https://www.wikidata.org/wiki/Q190391","display_name":"Central limit theorem","level":2,"score":0.482699990272522},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.40639999508857727},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.38199999928474426},{"id":"https://openalex.org/C202615002","wikidata":"https://www.wikidata.org/wiki/Q783507","display_name":"Differentiable function","level":2,"score":0.3578000068664551},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.35440000891685486},{"id":"https://openalex.org/C122048520","wikidata":"https://www.wikidata.org/wiki/Q2913954","display_name":"Percentile","level":2,"score":0.35339999198913574},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.3366999924182892},{"id":"https://openalex.org/C48406656","wikidata":"https://www.wikidata.org/wiki/Q534112","display_name":"Martingale (probability theory)","level":2,"score":0.3271999955177307},{"id":"https://openalex.org/C65778772","wikidata":"https://www.wikidata.org/wiki/Q12345341","display_name":"Asymptotic distribution","level":3,"score":0.31380000710487366},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.3075000047683716},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.30709999799728394},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.2849999964237213},{"id":"https://openalex.org/C64812099","wikidata":"https://www.wikidata.org/wiki/Q176604","display_name":"Random matrix","level":3,"score":0.27570000290870667},{"id":"https://openalex.org/C5297727","wikidata":"https://www.wikidata.org/wiki/Q786970","display_name":"Autocorrelation","level":2,"score":0.273499995470047},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.273499995470047},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.27320000529289246},{"id":"https://openalex.org/C101104100","wikidata":"https://www.wikidata.org/wiki/Q1063540","display_name":"Heteroscedasticity","level":2,"score":0.2662000060081482},{"id":"https://openalex.org/C162392398","wikidata":"https://www.wikidata.org/wiki/Q272404","display_name":"Finite set","level":2,"score":0.2590000033378601},{"id":"https://openalex.org/C111350023","wikidata":"https://www.wikidata.org/wiki/Q1191869","display_name":"Markov chain Monte Carlo","level":3,"score":0.25450000166893005},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.25209999084472656},{"id":"https://openalex.org/C207609745","wikidata":"https://www.wikidata.org/wiki/Q4944086","display_name":"Bootstrapping (finance)","level":2,"score":0.25049999356269836}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.12410","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12410","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.12410","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12410","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.8226521015167236,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,42,96],"propose":[1],"and":[2,57,80,110,134,157,161,181],"analyze":[3],"a":[4,23,53,68,81],"model-based":[5],"bootstrap":[6,48,70,93,98,145,156],"for":[7,76,91,132],"transition":[8,49,94],"kernels":[9],"in":[10,28,51],"finite":[11],"controlled":[12],"Markov":[13],"chains":[14],"(CMCs)":[15],"with":[16],"possibly":[17],"nonstationary":[18],"or":[19],"history-dependent":[20],"control":[21],"policies,":[22],"setting":[24],"that":[25,142],"arises":[26],"naturally":[27],"offline":[29,60,106],"reinforcement":[30],"learning":[31],"(RL)":[32],"when":[33],"the":[34,38,47,58,77,85,92,102,116,124,138,143,150,154,172],"behavior":[35],"policy":[36,107,112],"generating":[37],"data":[39],"is":[40],"unknown.":[41],"establish":[43],"distributional":[44,99],"consistency":[45,100],"of":[46,72,84,105,123],"estimator":[50],"both":[52],"single":[54],"long-chain":[55],"regime":[56],"episodic":[59,155],"RL":[61],"regime.":[62],"The":[63],"key":[64],"technical":[65],"tools":[66],"are":[67,162,174],"novel":[69,82],"law":[71],"large":[73],"numbers":[74],"(LLN)":[75],"visitation":[78],"counts":[79],"use":[83],"martingale":[86],"central":[87],"limit":[88],"theorem":[89],"(CLT)":[90],"increments.":[95],"extend":[97],"to":[101,165],"downstream":[103],"targets":[104],"evaluation":[108],"(OPE)":[109],"optimal":[111],"recovery":[113],"(OPR)":[114],"via":[115],"delta":[117],"method":[118],"by":[119],"verifying":[120],"Hadamard":[121],"differentiability":[122],"Bellman":[125],"operators,":[126],"yielding":[127],"asymptotically":[128],"valid":[129],"confidence":[130,146],"intervals":[131,147],"value":[133],"$Q$-functions.":[135],"Experiments":[136],"on":[137],"RiverSwim":[139],"problem":[140],"show":[141],"proposed":[144],"(CIs),":[148],"especially":[149],"percentile":[151],"CIs,":[152,160],"outperform":[153],"plug-in":[158],"CLT":[159],"often":[163],"close":[164],"nominal":[166],"($50\\%$,":[167],"$90\\%$,":[168],"$95\\%$)":[169],"coverage,":[170],"while":[171],"baselines":[173],"poorly":[175],"calibrated":[176],"at":[177],"small":[178],"sample":[179],"sizes":[180],"short":[182],"episode":[183],"lengths.":[184]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-14T00:00:00"}