{"id":"https://openalex.org/W3002435397","doi":"https://doi.org/10.1145/3336191.3371871","title":"Challenges, Best Practices and Pitfalls in Evaluating Results of Online Controlled Experiments","display_name":"Challenges, Best Practices and Pitfalls in Evaluating Results of Online Controlled Experiments","publication_year":2020,"publication_date":"2020-01-20","ids":{"openalex":"https://openalex.org/W3002435397","doi":"https://doi.org/10.1145/3336191.3371871","mag":"3002435397"},"language":"en","primary_location":{"id":"doi:10.1145/3336191.3371871","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3336191.3371871","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 13th International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039548707","display_name":"Somit Gupta","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Somit Gupta","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101666398","display_name":"Xiaolin Shi","orcid":"https://orcid.org/0000-0002-3705-1552"},"institutions":[{"id":"https://openalex.org/I4210142583","display_name":"Snap (United States)","ror":"https://ror.org/04dgkhg68","country_code":"US","type":"company","lineage":["https://openalex.org/I4210142583"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaolin Shi","raw_affiliation_strings":["Snap Inc., Santa Monica, CA, USA"],"affiliations":[{"raw_affiliation_string":"Snap Inc., Santa Monica, CA, USA","institution_ids":["https://openalex.org/I4210142583"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020697231","display_name":"Pavel Dmitriev","orcid":"https://orcid.org/0000-0001-5740-5146"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pavel Dmitriev","raw_affiliation_strings":["Outreach, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"Outreach, Seattle, WA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068016934","display_name":"Xin Fu","orcid":"https://orcid.org/0000-0003-4015-0778"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xin Fu","raw_affiliation_strings":["Facebook, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444","https://openalex.org/I4210099336"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102127831","display_name":"Avijit Mukherjee","orcid":null},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Avijit Mukherjee","raw_affiliation_strings":["Facebook, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444","https://openalex.org/I4210099336"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5039548707"],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.02149618,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"877","last_page":"880"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9776999950408936,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9776999950408936,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11443","display_name":"Advanced Statistical Process Monitoring","score":0.9749000072479248,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11798","display_name":"Optimal Experimental Design Methods","score":0.911899983882904,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7800639271736145},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.637547492980957},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6193071603775024},{"id":"https://openalex.org/keywords/harm","display_name":"Harm","score":0.5926721096038818},{"id":"https://openalex.org/keywords/best-practice","display_name":"Best practice","score":0.5563002228736877},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5555838346481323},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5356189012527466},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.529773473739624},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.4620373249053955},{"id":"https://openalex.org/keywords/outcome","display_name":"Outcome (game theory)","score":0.4496397376060486},{"id":"https://openalex.org/keywords/product","display_name":"Product (mathematics)","score":0.42338842153549194},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.33394917845726013},{"id":"https://openalex.org/keywords/risk-analysis","display_name":"Risk analysis (engineering)","score":0.3257651627063751},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.14080211520195007}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7800639271736145},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.637547492980957},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6193071603775024},{"id":"https://openalex.org/C2777363581","wikidata":"https://www.wikidata.org/wiki/Q15098235","display_name":"Harm","level":2,"score":0.5926721096038818},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.5563002228736877},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5555838346481323},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5356189012527466},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.529773473739624},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4620373249053955},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.4496397376060486},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.42338842153549194},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33394917845726013},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3257651627063751},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.14080211520195007},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3336191.3371871","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3336191.3371871","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 13th International Conference on Web Search and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5899999737739563,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W2114968482","https://openalex.org/W2126002144","https://openalex.org/W2169113736","https://openalex.org/W2468478068","https://openalex.org/W2509295096","https://openalex.org/W2517816274","https://openalex.org/W2584778415","https://openalex.org/W2616605651","https://openalex.org/W2740607000","https://openalex.org/W2743849295","https://openalex.org/W2945720420","https://openalex.org/W2946387282","https://openalex.org/W2954602266","https://openalex.org/W2965222965"],"related_works":["https://openalex.org/W2002383399","https://openalex.org/W644644594","https://openalex.org/W2506073049","https://openalex.org/W642180557","https://openalex.org/W2978999882","https://openalex.org/W2780591772","https://openalex.org/W3122105723","https://openalex.org/W1593090812","https://openalex.org/W4225469545","https://openalex.org/W2232462886"],"abstract_inverted_index":{"A/B":[0,59,150],"Testing":[1],"is":[2,25,61,100,124],"the":[3,8,29,67,94,97,101,116],"gold":[4],"standard":[5],"to":[6,31,42,51,83,92,108,148],"estimate":[7],"causal":[9],"relationship":[10],"between":[11],"a":[12,15,86,125],"change":[13,38,41],"in":[14,28,168],"product":[16],"and":[17,85,128,139,142,152,166,177],"its":[18],"impact":[19],"on":[20,173],"key":[21,56,98],"outcome":[22,74],"measures.":[23],"It":[24,198],"widely":[26],"used":[27],"industry":[30],"test":[32],"changes":[33,45],"ranging":[34],"from":[35,154],"simple":[36],"copy":[37],"or":[39],"UI":[40],"more":[43,138,140],"complex":[44],"like":[46],"using":[47],"machine":[48],"learning":[49],"models":[50],"personalize":[52],"user":[53],"experience.":[54],"The":[55],"aspect":[57],"of":[58,63,70,89,121,188],"testing":[60,151],"evaluation":[62],"experiment":[64,122,170],"results.":[65],"Designing":[66],"right":[68],"set":[69,88],"metrics":[71,91,123],"-":[72],"correct":[73],"measures,":[75],"data":[76],"quality":[77],"indicators,":[78],"guardrails":[79],"that":[80],"prevent":[81],"harm":[82],"business,":[84],"comprehensive":[87],"supporting":[90],"understand":[93],"\"why\"":[95],"behind":[96],"movements":[99],"#1":[102],"challenge":[103],"practitioners":[104],"face":[105],"when":[106],"trying":[107,147],"scale":[109],"their":[110],"experimentation":[111],"program":[112],"[11,":[113],"14].":[114],"On":[115],"technical":[117],"side,":[118],"improving":[119],"sensitivity":[120],"hard":[126],"problem":[127],"an":[129],"active":[130],"research":[131,184],"area,":[132],"with":[133],"large":[134],"practical":[135,178],"implications":[136],"as":[137,180,182],"small":[141],"medium":[143],"size":[144],"businesses":[145],"are":[146],"adopt":[149],"suffer":[153],"insufficient":[155],"power.":[156],"In":[157],"this":[158,189],"tutorial":[159,190],"we":[160],"will":[161],"discuss":[162],"challenges,":[163],"best":[164],"practices,":[165],"pitfalls":[167],"evaluating":[169],"results,":[171],"focusing":[172],"both":[174],"lessons":[175],"learned":[176],"guidelines":[179],"well":[181],"open":[183],"questions.":[185],"A":[186],"version":[187],"was":[191,199],"also":[192],"present":[193],"at":[194],"KDD":[195],"2019":[196],"[23].":[197],"attended":[200],"by":[201],"around":[202],"150":[203],"participants.":[204]},"counts_by_year":[],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}