{"id":"https://openalex.org/W2784068709","doi":"https://doi.org/10.1145/3159652.3159687","title":"Offline A/B Testing for Recommender Systems","display_name":"Offline A/B Testing for Recommender Systems","publication_year":2018,"publication_date":"2018-02-02","ids":{"openalex":"https://openalex.org/W2784068709","doi":"https://doi.org/10.1145/3159652.3159687","mag":"2784068709"},"language":"en","primary_location":{"id":"doi:10.1145/3159652.3159687","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3159652.3159687","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Eleventh ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1801.07030","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Alexandre Gilotte","orcid":null},"institutions":[{"id":"https://openalex.org/I4210161401","display_name":"Criteo (France)","ror":"https://ror.org/04vyg0r47","country_code":"FR","type":"company","lineage":["https://openalex.org/I4210161401"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Alexandre Gilotte","raw_affiliation_strings":["Criteo Research, Paris, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Criteo Research, Paris, France","institution_ids":["https://openalex.org/I4210161401"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Cl\u00e9ment Calauz\u00e8nes","orcid":null},"institutions":[{"id":"https://openalex.org/I4210161401","display_name":"Criteo (France)","ror":"https://ror.org/04vyg0r47","country_code":"FR","type":"company","lineage":["https://openalex.org/I4210161401"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Cl\u00e9ment Calauz\u00e8nes","raw_affiliation_strings":["Criteo Research, Paris, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Criteo Research, Paris, France","institution_ids":["https://openalex.org/I4210161401"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Thomas Nedelec","orcid":null},"institutions":[{"id":"https://openalex.org/I4210161401","display_name":"Criteo (France)","ror":"https://ror.org/04vyg0r47","country_code":"FR","type":"company","lineage":["https://openalex.org/I4210161401"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Thomas Nedelec","raw_affiliation_strings":["Criteo Research, Paris, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Criteo Research, Paris, France","institution_ids":["https://openalex.org/I4210161401"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Alexandre Abraham","orcid":null},"institutions":[{"id":"https://openalex.org/I4210161401","display_name":"Criteo (France)","ror":"https://ror.org/04vyg0r47","country_code":"FR","type":"company","lineage":["https://openalex.org/I4210161401"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Alexandre Abraham","raw_affiliation_strings":["Criteo Research, Paris, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Criteo Research, Paris, France","institution_ids":["https://openalex.org/I4210161401"]}]},{"author_position":"last","author":{"id":null,"display_name":"Simon Doll\u00e9","orcid":null},"institutions":[{"id":"https://openalex.org/I4210161401","display_name":"Criteo (France)","ror":"https://ror.org/04vyg0r47","country_code":"FR","type":"company","lineage":["https://openalex.org/I4210161401"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Simon Doll\u00e9","raw_affiliation_strings":["Criteo Research, Paris, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Criteo Research, Paris, France","institution_ids":["https://openalex.org/I4210161401"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":20.1542,"has_fulltext":false,"cited_by_count":130,"citation_normalized_percentile":{"value":0.99289752,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"198","last_page":"206"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.9663000106811523,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.9517999887466431},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.703000009059906},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.6962000131607056},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.6158999800682068},{"id":"https://openalex.org/keywords/product","display_name":"Product (mathematics)","score":0.49070000648498535},{"id":"https://openalex.org/keywords/thompson-sampling","display_name":"Thompson sampling","score":0.43970000743865967},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.3937999904155731},{"id":"https://openalex.org/keywords/online-and-offline","display_name":"Online and offline","score":0.38600000739097595},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.34779998660087585}],"concepts":[{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.9517999887466431},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7487999796867371},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.703000009059906},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.6962000131607056},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.6158999800682068},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.49070000648498535},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4706999957561493},{"id":"https://openalex.org/C73602740","wikidata":"https://www.wikidata.org/wiki/Q7795822","display_name":"Thompson sampling","level":3,"score":0.43970000743865967},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4047999978065491},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3937999904155731},{"id":"https://openalex.org/C2780102126","wikidata":"https://www.wikidata.org/wiki/Q10928179","display_name":"Online and offline","level":2,"score":0.38600000739097595},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38350000977516174},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.34779998660087585},{"id":"https://openalex.org/C21569690","wikidata":"https://www.wikidata.org/wiki/Q94702","display_name":"Collaborative filtering","level":3,"score":0.3440999984741211},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.34380000829696655},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3416000008583069},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.33899998664855957},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.326200008392334},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.30399999022483826},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.2994999885559082},{"id":"https://openalex.org/C196921405","wikidata":"https://www.wikidata.org/wiki/Q786431","display_name":"Online algorithm","level":2,"score":0.29319998621940613},{"id":"https://openalex.org/C2780490138","wikidata":"https://www.wikidata.org/wiki/Q7079636","display_name":"Offline learning","level":3,"score":0.2906000018119812},{"id":"https://openalex.org/C16910744","wikidata":"https://www.wikidata.org/wiki/Q7705759","display_name":"Test data","level":2,"score":0.2759999930858612},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.2694999873638153},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.2581999897956848},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2531000077724457}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3159652.3159687","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3159652.3159687","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Eleventh ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1801.07030","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1801.07030","pdf_url":"https://arxiv.org/pdf/1801.07030","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-02457457v1","is_oa":true,"landing_page_url":"https://hal.science/hal-02457457","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"WSDM '18 - The 11th ACM International Conference on Web Search and Data Mining, Feb 2018, Los Angeles, United States. pp.198-206, ⟨10.1145/3159652.3159687⟩","raw_type":"Conference papers"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1801.07030","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1801.07030","pdf_url":"https://arxiv.org/pdf/1801.07030","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W1971040550","https://openalex.org/W1971956681","https://openalex.org/W1992549066","https://openalex.org/W2005415325","https://openalex.org/W2033442452","https://openalex.org/W2055598786","https://openalex.org/W2100119435","https://openalex.org/W2119717200","https://openalex.org/W2134445663","https://openalex.org/W2138909795","https://openalex.org/W2219888463","https://openalex.org/W4206038828","https://openalex.org/W4233471163","https://openalex.org/W4251560691","https://openalex.org/W6601741120"],"related_works":[],"abstract_inverted_index":{"Online":[0],"A/B":[1,175],"testing":[2,19],"evaluates":[3],"the":[4,26,29,64,82,134,145,162],"impact":[5],"of":[6,25,28,81,85,161],"a":[7,14,23,33,38,89,153,159,178],"new":[8,49,90,154],"technology":[9],"by":[10,88,137,172],"running":[11,173],"it":[12,143],"in":[13,56,73,144],"real":[15],"production":[16],"environment":[17],"and":[18,51],"its":[20],"performance":[21,86,93],"on":[22,42,48,123,177],"subset":[24],"users":[27],"platform.":[30],"It":[31],"is":[32],"well-known":[34],"practice":[35],"to":[36,45,52,58,151],"run":[37],"preliminary":[39],"offline":[40,68,78],"evaluation":[41],"historical":[43],"data":[44],"iterate":[46],"faster":[47],"ideas,":[50],"detect":[53],"poor":[54],"policies":[55],"order":[57],"avoid":[59],"losing":[60],"money":[61],"or":[62,102,113],"breaking":[63],"system.":[65,182],"For":[66],"such":[67,108],"evaluations,":[69],"we":[70,132],"are":[71],"interested":[72],"methods":[74],"that":[75],"can":[76,94],"compute":[77],"an":[79],"estimate":[80],"potential":[83],"uplift":[84],"generated":[87],"technology.":[91],"Offline":[92],"be":[95],"measured":[96],"using":[97],"estimators":[98,139,164],"known":[99],"as":[100,109],"counterfactual":[101,106,155],"off-policy":[103],"estimators.":[104],"Traditional":[105],"estimators,":[107],"capped":[110],"importance":[111,115],"sampling":[112],"normalised":[114],"sampling,":[116],"exhibit":[117],"unsatisfying":[118],"bias-variance":[119],"compromises":[120],"when":[121],"experimenting":[122],"personalized":[124],"product":[125],"recommendation":[126],"systems.":[127],"To":[128],"overcome":[129],"this":[130],"issue,":[131],"model":[133],"bias":[135],"incurred":[136],"these":[138],"rather":[140],"than":[141],"bound":[142],"worst":[146],"case,":[147],"which":[148],"leads":[149],"us":[150],"propose":[152],"estimator.":[156],"We":[157],"provide":[158],"benchmark":[160],"different":[163],"showing":[165],"their":[166],"correlation":[167],"with":[168],"business":[169],"metrics":[170],"observed":[171],"online":[174],"tests":[176],"large-scale":[179],"commercial":[180],"recommender":[181]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":20},{"year":2024,"cited_by_count":17},{"year":2023,"cited_by_count":19},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":13},{"year":2020,"cited_by_count":19},{"year":2019,"cited_by_count":16},{"year":2018,"cited_by_count":1},{"year":2012,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2018-01-26T00:00:00"}