{"id":"https://openalex.org/W3124872564","doi":"https://doi.org/10.1145/3460231.3474231","title":"Debiased Off-Policy Evaluation for Recommendation Systems","display_name":"Debiased Off-Policy Evaluation for Recommendation Systems","publication_year":2021,"publication_date":"2021-09-13","ids":{"openalex":"https://openalex.org/W3124872564","doi":"https://doi.org/10.1145/3460231.3474231","mag":"3124872564"},"language":"en","primary_location":{"id":"doi:10.1145/3460231.3474231","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3460231.3474231","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Fifteenth ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2002.08536","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102762244","display_name":"Yusuke Narita","orcid":"https://orcid.org/0000-0003-0314-3384"},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Yusuke Narita","raw_affiliation_strings":["Yale University, United States","Yale University, United#N#                            States"],"affiliations":[{"raw_affiliation_string":"Yale University, United States","institution_ids":["https://openalex.org/I32971472"]},{"raw_affiliation_string":"Yale University, United#N#                            States","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033866977","display_name":"Shota Yasui","orcid":null},"institutions":[{"id":"https://openalex.org/I4210089607","display_name":"CyberAgent (Japan)","ror":"https://ror.org/0060jg679","country_code":"JP","type":"company","lineage":["https://openalex.org/I4210089607"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Shota Yasui","raw_affiliation_strings":["AILab CyberAgent, Inc., Japan"],"affiliations":[{"raw_affiliation_string":"AILab CyberAgent, Inc., Japan","institution_ids":["https://openalex.org/I4210089607"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028705026","display_name":"Kohei Yata","orcid":null},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kohei Yata","raw_affiliation_strings":["Department of Economics Yale University, United States"],"affiliations":[{"raw_affiliation_string":"Department of Economics Yale University, United States","institution_ids":["https://openalex.org/I32971472"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5102762244"],"corresponding_institution_ids":["https://openalex.org/I32971472"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.01871601,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"372","last_page":"379"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10603","display_name":"Smart Grid Energy Management","score":0.9876999855041504,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9751999974250793,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.8784419298171997},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.7409650087356567},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6769893169403076},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.6720946431159973},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6691064238548279},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.4885433614253998},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.4770968556404114},{"id":"https://openalex.org/keywords/property","display_name":"Property (philosophy)","score":0.4499271810054779},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.4337024688720703},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4067828357219696},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38311535120010376},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3458935022354126},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.22797131538391113},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1801270842552185}],"concepts":[{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.8784419298171997},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.7409650087356567},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6769893169403076},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.6720946431159973},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6691064238548279},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.4885433614253998},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.4770968556404114},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.4499271810054779},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.4337024688720703},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4067828357219696},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38311535120010376},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3458935022354126},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.22797131538391113},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1801270842552185},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1145/3460231.3474231","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3460231.3474231","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Fifteenth ACM Conference on Recommender Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2002.08536","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2002.08536","pdf_url":"https://arxiv.org/pdf/2002.08536","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3124872564","is_oa":false,"landing_page_url":"https://EconPapers.repec.org/RePEc:arx:papers:2002.08536","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"RePEc: Research Papers in Economics","raw_type":null},{"id":"pmh:oai:RePEc:arx:papers:2002.08536","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"preprint"},{"id":"doi:10.48550/arxiv.2002.08536","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2002.08536","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2002.08536","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2002.08536","pdf_url":"https://arxiv.org/pdf/2002.08536","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/8","score":0.5799999833106995,"display_name":"Decent work and economic growth"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":5,"referenced_works":["https://openalex.org/W1998427280","https://openalex.org/W2112420033","https://openalex.org/W2159927689","https://openalex.org/W2890951405","https://openalex.org/W3123436326"],"related_works":["https://openalex.org/W3200267675","https://openalex.org/W2368912315","https://openalex.org/W1511670714","https://openalex.org/W3136946427","https://openalex.org/W3104349857","https://openalex.org/W3099420497","https://openalex.org/W3044556022","https://openalex.org/W2919861672","https://openalex.org/W2989664601","https://openalex.org/W2886249230","https://openalex.org/W3026534141","https://openalex.org/W2884412095","https://openalex.org/W3035302362","https://openalex.org/W940094962","https://openalex.org/W2980012679","https://openalex.org/W3135386061","https://openalex.org/W2955989140","https://openalex.org/W1483524195","https://openalex.org/W3118447610","https://openalex.org/W3208987033"],"abstract_inverted_index":{"Efficient":[0],"methods":[1],"to":[2,72,97,108,153],"evaluate":[3],"new":[4],"algorithms":[5,48],"are":[6,22,25,135],"critical":[7],"for":[8],"improving":[9],"interactive":[10],"bandit":[11],"and":[12,27,29],"reinforcement":[13,147],"learning":[14],"systems":[15],"such":[16],"as":[17,85],"recommendation":[18],"systems.":[19],"A/B":[20],"tests":[21],"reliable,":[23],"but":[24],"time-":[26],"money-consuming,":[28],"entail":[30],"a":[31,58,77,81,94,127,143,158],"risk":[32],"of":[33,47,76,83,101,130],"failure.":[34],"In":[35],"this":[36],"paper,":[37],"we":[38],"develop":[39],"an":[40],"alternative":[41],"method,":[42],"which":[43,125],"predicts":[44],"the":[45,64,73,86,99,106,110,113,120],"performance":[46,75],"given":[49],"historical":[50],"data":[51],"that":[52,66,164],"may":[53],"have":[54],"been":[55],"generated":[56],"by":[57,142,157],"different":[59],"algorithm.":[60],"Our":[61],"estimator":[62],"has":[63],"property":[65],"its":[67],"prediction":[68],"converges":[69],"in":[70,112],"probability":[71],"true":[74],"counterfactual":[78],"algorithm":[79],"at":[80],"rate":[82],",":[84],"sample":[87],"size":[88],"N":[89],"increases.":[90],"We":[91,138,149,162],"also":[92],"show":[93],"correct":[95],"way":[96],"estimate":[98],"variance":[100],"our":[102,140,165],"prediction,":[103],"thus":[104],"allowing":[105],"analyst":[107,121],"quantify":[109],"uncertainty":[111],"prediction.":[114],"These":[115],"properties":[116],"hold":[117],"even":[118],"when":[119],"does":[122],"not":[123],"know":[124],"among":[126],"large":[128],"number":[129],"potentially":[131],"important":[132],"state":[133],"variables":[134],"actually":[136],"important.":[137],"validate":[139],"method":[141,166],"simulation":[144],"experiment":[145],"about":[146],"learning.":[148],"finally":[150],"apply":[151],"it":[152],"improve":[154],"advertisement":[155,160],"design":[156],"major":[159],"company.":[161],"find":[163],"produces":[167],"smaller":[168],"mean":[169],"squared":[170],"errors":[171],"than":[172],"state-of-the-art":[173],"methods.":[174]},"counts_by_year":[],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}