{"id":"https://openalex.org/W2298667432","doi":"https://doi.org/10.1109/tac.2018.2799521","title":"On the Whittle Index for Restless Multiarmed Hidden Markov Bandits","display_name":"On the Whittle Index for Restless Multiarmed Hidden Markov Bandits","publication_year":2018,"publication_date":"2018-01-30","ids":{"openalex":"https://openalex.org/W2298667432","doi":"https://doi.org/10.1109/tac.2018.2799521","mag":"2298667432"},"language":"en","primary_location":{"id":"doi:10.1109/tac.2018.2799521","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tac.2018.2799521","pdf_url":null,"source":{"id":"https://openalex.org/S184954342","display_name":"IEEE Transactions on Automatic Control","issn_l":"0018-9286","issn":["0018-9286","1558-2523","2334-3303"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Automatic Control","raw_type":"journal-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1603.04739","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052401633","display_name":"Rahul Meshram","orcid":"https://orcid.org/0000-0003-3966-3269"},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]},{"id":"https://openalex.org/I4210144315","display_name":"Bharti Centre for Communication","ror":"https://ror.org/057rcgy56","country_code":"IN","type":"other","lineage":["https://openalex.org/I4210144315"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"Rahul Meshram","raw_affiliation_strings":["Department of Electrical Engineering and the Bharti Centre for Communication, Indian Institute of Technology Bombay, Mumbai, India"],"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and the Bharti Centre for Communication, Indian Institute of Technology Bombay, Mumbai, India","institution_ids":["https://openalex.org/I162827531","https://openalex.org/I4210144315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017359920","display_name":"D. Manjunath","orcid":"https://orcid.org/0000-0001-7302-284X"},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]},{"id":"https://openalex.org/I4210144315","display_name":"Bharti Centre for Communication","ror":"https://ror.org/057rcgy56","country_code":"IN","type":"other","lineage":["https://openalex.org/I4210144315"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"D. Manjunath","raw_affiliation_strings":["Department of Electrical Engineering and the Bharti Centre for Communication, Indian Institute of Technology Bombay, Mumbai, India"],"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and the Bharti Centre for Communication, Indian Institute of Technology Bombay, Mumbai, India","institution_ids":["https://openalex.org/I162827531","https://openalex.org/I4210144315"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5030430086","display_name":"Aditya Gopalan","orcid":"https://orcid.org/0000-0002-7323-2975"},"institutions":[{"id":"https://openalex.org/I59270414","display_name":"Indian Institute of Science Bangalore","ror":"https://ror.org/04dese585","country_code":"IN","type":"education","lineage":["https://openalex.org/I59270414"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Aditya Gopalan","raw_affiliation_strings":["Department of Electrical Communication Engineering, Indian Institute of Science, Bangalore, India"],"affiliations":[{"raw_affiliation_string":"Department of Electrical Communication Engineering, Indian Institute of Science, Bangalore, India","institution_ids":["https://openalex.org/I59270414"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5052401633"],"corresponding_institution_ids":["https://openalex.org/I162827531","https://openalex.org/I4210144315"],"apc_list":null,"apc_paid":null,"fwci":0.4402,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":{"value":0.67426225,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"63","issue":"9","first_page":"3046","last_page":"3053"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10579","display_name":"Cognitive Radio Networks and Spectrum Sensing","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10603","display_name":"Smart Grid Energy Management","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/randomness","display_name":"Randomness","score":0.7090427279472351},{"id":"https://openalex.org/keywords/markov-chain","display_name":"Markov chain","score":0.6351537108421326},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.5419214367866516},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.5296370983123779},{"id":"https://openalex.org/keywords/thompson-sampling","display_name":"Thompson sampling","score":0.48441535234451294},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.4794784486293793},{"id":"https://openalex.org/keywords/multi-armed-bandit","display_name":"Multi-armed bandit","score":0.47884035110473633},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4448709189891815},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.41748902201652527},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.4116376042366028},{"id":"https://openalex.org/keywords/signal","display_name":"SIGNAL (programming language)","score":0.410910964012146},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.23231780529022217},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.21258684992790222},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.18947526812553406},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.14679580926895142}],"concepts":[{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.7090427279472351},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.6351537108421326},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.5419214367866516},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.5296370983123779},{"id":"https://openalex.org/C73602740","wikidata":"https://www.wikidata.org/wiki/Q7795822","display_name":"Thompson sampling","level":3,"score":0.48441535234451294},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4794784486293793},{"id":"https://openalex.org/C123197309","wikidata":"https://www.wikidata.org/wiki/Q2882343","display_name":"Multi-armed bandit","level":3,"score":0.47884035110473633},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4448709189891815},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.41748902201652527},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.4116376042366028},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.410910964012146},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.23231780529022217},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.21258684992790222},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.18947526812553406},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.14679580926895142},{"id":"https://openalex.org/C50817715","wikidata":"https://www.wikidata.org/wiki/Q79895177","display_name":"Regret","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0}],"mesh":[],"locations_count":7,"locations":[{"id":"doi:10.1109/tac.2018.2799521","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tac.2018.2799521","pdf_url":null,"source":{"id":"https://openalex.org/S184954342","display_name":"IEEE Transactions on Automatic Control","issn_l":"0018-9286","issn":["0018-9286","1558-2523","2334-3303"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Automatic Control","raw_type":"journal-article"},{"id":"pmh:oai:eprints.iisc.ac.in:60746","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4377196309","display_name":"NOT FOUND REPOSITORY (Indian Institute of Science Bangalore)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I59270414","host_organization_name":"Indian Institute of Science Bangalore","host_organization_lineage":["https://openalex.org/I59270414"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Journal Article"},{"id":"pmh:oai:arXiv.org:1603.04739","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1603.04739","pdf_url":"https://arxiv.org/pdf/1603.04739","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"mag:2298667432","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1603.04739.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"pmh:oai:dsapce.library.iitb.ac.in:100/23318","is_oa":false,"landing_page_url":"http://dspace.library.iitb.ac.in/xmlui/handle/100/23318","pdf_url":null,"source":{"id":"https://openalex.org/S4306400899","display_name":"DSpace (IIT Bombay)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I162827531","host_organization_name":"Indian Institute of Technology Bombay","host_organization_lineage":["https://openalex.org/I162827531"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"},{"id":"pmh:oai:dspace.library.iitb.ac.in:100/23318","is_oa":false,"landing_page_url":"http://doi.org/10.1002/2014GL060388","pdf_url":null,"source":{"id":"https://openalex.org/S4306400899","display_name":"DSpace (IIT Bombay)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I162827531","host_organization_name":"Indian Institute of Technology Bombay","host_organization_lineage":["https://openalex.org/I162827531"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"},{"id":"doi:10.48550/arxiv.1603.04739","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1603.04739","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1603.04739","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1603.04739","pdf_url":"https://arxiv.org/pdf/1603.04739","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.7099999785423279}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2298667432.pdf","grobid_xml":"https://content.openalex.org/works/W2298667432.grobid-xml"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W1702782474","https://openalex.org/W1970030516","https://openalex.org/W1985014315","https://openalex.org/W2003842604","https://openalex.org/W2028586524","https://openalex.org/W2038279556","https://openalex.org/W2056921512","https://openalex.org/W2063142364","https://openalex.org/W2069065005","https://openalex.org/W2087334858","https://openalex.org/W2098432798","https://openalex.org/W2101203346","https://openalex.org/W2104389751","https://openalex.org/W2105556121","https://openalex.org/W2114951919","https://openalex.org/W2118726574","https://openalex.org/W2121761793","https://openalex.org/W2124253904","https://openalex.org/W2134945900","https://openalex.org/W2141092858","https://openalex.org/W2141515329","https://openalex.org/W2142819538","https://openalex.org/W2153107031","https://openalex.org/W2153345187","https://openalex.org/W2153441348","https://openalex.org/W2155529408","https://openalex.org/W2156107305","https://openalex.org/W2171671264","https://openalex.org/W4394592908","https://openalex.org/W6698017823"],"related_works":["https://openalex.org/W2607012320","https://openalex.org/W2601096073","https://openalex.org/W112391091","https://openalex.org/W1553290137","https://openalex.org/W2141515329","https://openalex.org/W2185566361","https://openalex.org/W65148180","https://openalex.org/W2790469074","https://openalex.org/W2953067537","https://openalex.org/W2981378548","https://openalex.org/W2626766158","https://openalex.org/W2148250692","https://openalex.org/W2736645240","https://openalex.org/W2215479434","https://openalex.org/W2788664414","https://openalex.org/W2167555642","https://openalex.org/W2138657485","https://openalex.org/W2072043571","https://openalex.org/W2095160246","https://openalex.org/W2001154059"],"abstract_inverted_index":{"We":[0,159,187],"consider":[1],"a":[2,34,38,114,129,180,208,224],"restless":[3],"multiarmed":[4],"bandit":[5,164,210,222],"in":[6,12,83,128,135,155,168],"which":[7,194],"each":[8,67,70,73,136,237],"arm":[9,19,26,48,57,74,89,100],"can":[10],"be":[11,107],"one":[13],"of":[14,24,46,98,151,232],"two":[15],"states.":[16],"When":[17],"an":[18,172,213],"is":[20,27,49,53,58,64,90,101,127,179,198],"sampled,":[21],"the":[22,25,31,44,47,56,88,96,99,110,121,133,141,149,152,157,162,184,195,200,217,220,230,233,242],"state":[23,45,76,97],"not":[28,59,93],"available":[29,54],"to":[30,78,106,131,139],"sampler.":[32],"Instead,":[33],"binary":[35,116],"signal":[36,52],"with":[37],"known":[39,79],"randomness":[40],"that":[41,206],"depends":[42],"on":[43,86],"available.":[50],"No":[51],"if":[55],"sampled.":[60,94],"An":[61],"arm-dependent":[62],"reward":[63,182],"accrued":[65],"from":[66,109],"sampling.":[68],"In":[69],"time":[71,137],"step,":[72],"changes":[75],"according":[77],"transition":[80],"probabilities,":[81],"which,":[82],"turn,":[84],"depend":[85],"whether":[87],"sampled":[91],"or":[92],"Since":[95],"never":[102],"visible":[103],"and":[104,113,165],"has":[105],"inferred":[108],"current":[111],"belief":[112],"possible":[115],"signal,":[117],"we":[118,147,204,228],"call":[119],"this":[120],"hidden":[122],"Markov":[123],"bandit.":[124],"Our":[125],"interest":[126],"policy":[130,176,197],"select":[132],"arm(s)":[134],"step":[138],"maximize":[140],"infinite":[142],"horizon":[143],"discounted":[144],"reward.":[145],"Specifically,":[146],"seek":[148],"use":[150],"Whittle":[153,234],"index":[154,235],"selecting":[156],"arms.":[158],"first":[160],"analyze":[161],"single-armed":[163,209,221],"show":[166,205],"that,":[167],"general,":[169],"it":[170],"admits":[171,223],"approximate":[173],"threshold-type":[174,225],"optimal":[175,201,226],"when":[177,219],"there":[178],"positive":[181],"for":[183,193,236],"\u201cno-sample\u201d":[185],"action.":[186],"also":[188,211],"identify":[189],"several":[190],"special":[191],"cases":[192],"threshold":[196],"indeed":[199],"policy.":[202],"Next,":[203],"such":[207],"satisfies":[212],"approximate-indexability":[214],"property.":[215],"For":[216],"case":[218],"policy,":[227],"perform":[229],"calculation":[231],"arm.":[238],"Numerical":[239],"examples":[240],"illustrate":[241],"analytical":[243],"results.":[244]},"counts_by_year":[{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}