{"id":"https://openalex.org/W3044963235","doi":"https://doi.org/10.1145/3394486.3403229","title":"Counterfactual Evaluation of Slate Recommendations with Sequential Reward Interactions","display_name":"Counterfactual Evaluation of Slate Recommendations with Sequential Reward Interactions","publication_year":2020,"publication_date":"2020-08-20","ids":{"openalex":"https://openalex.org/W3044963235","doi":"https://doi.org/10.1145/3394486.3403229","mag":"3044963235"},"language":"en","primary_location":{"id":"doi:10.1145/3394486.3403229","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3394486.3403229","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2007.12986","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037761875","display_name":"James O. McInerney","orcid":"https://orcid.org/0000-0003-1885-2503"},"institutions":[{"id":"https://openalex.org/I869089601","display_name":"Netflix (United States)","ror":"https://ror.org/0197qw696","country_code":"US","type":"company","lineage":["https://openalex.org/I869089601"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James McInerney","raw_affiliation_strings":["Netflix, Los Gatos, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Netflix, Los Gatos, CA, USA","institution_ids":["https://openalex.org/I869089601"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074347474","display_name":"Brian M. Brost","orcid":"https://orcid.org/0000-0002-9810-3950"},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Brian Brost","raw_affiliation_strings":["Spotify, New York, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Spotify, New York, NY, USA","institution_ids":["https://openalex.org/I4210122154"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020257829","display_name":"Praveen Chandar","orcid":"https://orcid.org/0009-0008-2199-2631"},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Praveen Chandar","raw_affiliation_strings":["Spotify, New York, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Spotify, New York, NY, USA","institution_ids":["https://openalex.org/I4210122154"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018503243","display_name":"Rishabh Mehrotra","orcid":"https://orcid.org/0000-0002-0836-4605"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rishabh Mehrotra","raw_affiliation_strings":["Spotify, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Spotify, London, United Kingdom","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5109487140","display_name":"Benjamin Carterette","orcid":null},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Benjamin Carterette","raw_affiliation_strings":["Spotify, New York, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Spotify, New York, NY, USA","institution_ids":["https://openalex.org/I4210122154"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":46,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1779","last_page":"1788"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.9465097784996033},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7929918169975281},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.6764410734176636},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.6127037405967712},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.5688864588737488},{"id":"https://openalex.org/keywords/independence","display_name":"Independence (probability theory)","score":0.5356433391571045},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.4173853397369385},{"id":"https://openalex.org/keywords/counterfactual-conditional","display_name":"Counterfactual conditional","score":0.4124351739883423},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.364467591047287},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3555642366409302},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.3548126220703125},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.1261349320411682},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12049853801727295},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.08773940801620483}],"concepts":[{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.9465097784996033},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7929918169975281},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.6764410734176636},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.6127037405967712},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.5688864588737488},{"id":"https://openalex.org/C35651441","wikidata":"https://www.wikidata.org/wiki/Q625303","display_name":"Independence (probability theory)","level":2,"score":0.5356433391571045},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.4173853397369385},{"id":"https://openalex.org/C71889745","wikidata":"https://www.wikidata.org/wiki/Q1783264","display_name":"Counterfactual conditional","level":3,"score":0.4124351739883423},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.364467591047287},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3555642366409302},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3548126220703125},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1261349320411682},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12049853801727295},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.08773940801620483},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3394486.3403229","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3394486.3403229","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2007.12986","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2007.12986","pdf_url":"https://arxiv.org/pdf/2007.12986","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2007.12986","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2007.12986","pdf_url":"https://arxiv.org/pdf/2007.12986","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":32,"referenced_works":["https://openalex.org/W1886704267","https://openalex.org/W2020631728","https://openalex.org/W2022995284","https://openalex.org/W2045745608","https://openalex.org/W2112420033","https://openalex.org/W2112508839","https://openalex.org/W2188353343","https://openalex.org/W2279176662","https://openalex.org/W2340679559","https://openalex.org/W2406454855","https://openalex.org/W2603530220","https://openalex.org/W2783272285","https://openalex.org/W2784068709","https://openalex.org/W2795199972","https://openalex.org/W2798460079","https://openalex.org/W2897183834","https://openalex.org/W2902040508","https://openalex.org/W2902572901","https://openalex.org/W2906762886","https://openalex.org/W2911829170","https://openalex.org/W2944832664","https://openalex.org/W2951431594","https://openalex.org/W2953633867","https://openalex.org/W2962694783","https://openalex.org/W2962803570","https://openalex.org/W2964297722","https://openalex.org/W3099420497","https://openalex.org/W3116843828","https://openalex.org/W4301884499","https://openalex.org/W4310895557","https://openalex.org/W6680885310","https://openalex.org/W6737893269"],"related_works":["https://openalex.org/W2056582926","https://openalex.org/W3137864021","https://openalex.org/W2162910442","https://openalex.org/W2079879923","https://openalex.org/W4200271736","https://openalex.org/W2104420793","https://openalex.org/W3017854570","https://openalex.org/W2028689793","https://openalex.org/W4313936361","https://openalex.org/W4242448314"],"abstract_inverted_index":{"Users":[0],"of":[1,24,82,101,126],"music":[2],"streaming,":[3,5],"video":[4],"news":[6],"recommendation,":[7],"and":[8,20,111,128],"e-commerce":[9],"services":[10],"often":[11],"engage":[12],"with":[13,65],"content":[14],"in":[15,62,68,89,93,109,124],"a":[16,28,53,94,113],"sequential":[17,60,133],"manner.":[18,72],"Providing":[19],"evaluating":[21],"good":[22],"sequences":[23],"recommendations":[25,135],"is":[26],"therefore":[27],"central":[29],"problem":[30],"for":[31,59,131],"these":[32],"services.":[33],"Prior":[34],"reweighting-based":[35],"counterfactual":[36,55],"evaluation":[37],"methods":[38,123],"either":[39],"suffer":[40],"from":[41],"high":[42],"variance":[43,67],"or":[44],"make":[45],"strong":[46],"independence":[47],"assumptions":[48,77],"about":[49,78],"rewards.":[50],"We":[51],"propose":[52],"new":[54],"estimator":[56],"that":[57,96,118],"allows":[58],"interactions":[61],"the":[63,79,83,87,90,98,104,132],"rewards":[64,88,102],"lower":[66],"an":[69],"asymptotically":[70],"unbiased":[71],"Our":[73],"method":[74],"uses":[75],"graphical":[76],"causal":[80],"relationships":[81],"slate":[84],"to":[85],"reweight":[86],"logging":[91],"policy":[92],"way":[95],"approximates":[97],"expected":[99],"sum":[100],"under":[103],"target":[105],"policy.":[106],"Extensive":[107],"experiments":[108],"simulation":[110],"on":[112],"live":[114],"recommender":[115],"system":[116],"show":[117],"our":[119],"approach":[120],"outperforms":[121],"existing":[122],"terms":[125],"bias":[127],"data":[129],"efficiency":[130],"track":[134],"problem.":[136]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}