{"id":"https://openalex.org/W2965222965","doi":"https://doi.org/10.1145/3292500.3332297","title":"Challenges, Best Practices and Pitfalls in Evaluating Results of Online Controlled Experiments","display_name":"Challenges, Best Practices and Pitfalls in Evaluating Results of Online Controlled Experiments","publication_year":2019,"publication_date":"2019-07-25","ids":{"openalex":"https://openalex.org/W2965222965","doi":"https://doi.org/10.1145/3292500.3332297","mag":"2965222965"},"language":"en","primary_location":{"id":"doi:10.1145/3292500.3332297","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3292500.3332297","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101666398","display_name":"Xiaolin Shi","orcid":"https://orcid.org/0000-0002-3705-1552"},"institutions":[{"id":"https://openalex.org/I4210142583","display_name":"Snap (United States)","ror":"https://ror.org/04dgkhg68","country_code":"US","type":"company","lineage":["https://openalex.org/I4210142583"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Xiaolin Shi","raw_affiliation_strings":["Snap Inc., Santa Monica, CA, USA"],"affiliations":[{"raw_affiliation_string":"Snap Inc., Santa Monica, CA, USA","institution_ids":["https://openalex.org/I4210142583"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020697231","display_name":"Pavel Dmitriev","orcid":"https://orcid.org/0000-0001-5740-5146"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pavel Dmitriev","raw_affiliation_strings":["Outreach, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"Outreach, Seattle, WA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039548707","display_name":"Somit Gupta","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Somit Gupta","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068016934","display_name":"Xin Fu","orcid":"https://orcid.org/0000-0003-4015-0778"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xin Fu","raw_affiliation_strings":["Facebook, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444","https://openalex.org/I4210099336"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5101666398"],"corresponding_institution_ids":["https://openalex.org/I4210142583"],"apc_list":null,"apc_paid":null,"fwci":1.6388,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.85661179,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"3189","last_page":"3190"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9498000144958496,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9498000144958496,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9409999847412109,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.9307000041007996,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6134195327758789}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6134195327758789}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3292500.3332297","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3292500.3332297","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5799999833106995,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W1500478607","https://openalex.org/W1896407793","https://openalex.org/W1982530130","https://openalex.org/W2023841894","https://openalex.org/W2030660637","https://openalex.org/W2074380434","https://openalex.org/W2080770071","https://openalex.org/W2114968482","https://openalex.org/W2126002144","https://openalex.org/W2158254843","https://openalex.org/W2169113736","https://openalex.org/W2170443290","https://openalex.org/W2468478068","https://openalex.org/W2509860763","https://openalex.org/W2532610538","https://openalex.org/W2584444538","https://openalex.org/W2584778415","https://openalex.org/W2607670480","https://openalex.org/W2740607000","https://openalex.org/W2743849295","https://openalex.org/W2809468631","https://openalex.org/W2884019407","https://openalex.org/W2897260156","https://openalex.org/W2897477542","https://openalex.org/W2945720420","https://openalex.org/W2946387282","https://openalex.org/W2952127798","https://openalex.org/W2954602266","https://openalex.org/W2963366444","https://openalex.org/W3103282043","https://openalex.org/W3104418946"],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2093578348","https://openalex.org/W2376932109","https://openalex.org/W2382290278","https://openalex.org/W2350741829","https://openalex.org/W3004735627","https://openalex.org/W2130043461"],"abstract_inverted_index":{"A/B":[0,59,150],"Testing":[1],"is":[2,25,61,100,124],"the":[3,8,29,67,94,97,101,116],"gold":[4],"standard":[5],"to":[6,31,42,51,83,92,108,148],"estimate":[7],"causal":[9],"relationship":[10],"between":[11],"a":[12,15,86,125],"change":[13,38,41],"in":[14,28,168],"product":[16],"and":[17,85,128,139,142,152,166,177],"its":[18],"impact":[19],"on":[20,173],"key":[21,56,98],"outcome":[22,74],"measures.":[23],"It":[24],"widely":[26],"used":[27],"industry":[30],"test":[32],"changes":[33,45],"ranging":[34],"from":[35,154],"simple":[36],"copy":[37],"or":[39],"UI":[40],"more":[43,138,140],"complex":[44],"like":[46],"using":[47],"machine":[48],"learning":[49],"models":[50],"personalize":[52],"user":[53],"experience.":[54],"The":[55],"aspect":[57],"of":[58,63,70,89,121],"testing":[60,151],"evaluation":[62],"experiment":[64,122,170],"results.":[65],"Designing":[66],"right":[68],"set":[69,88],"metrics":[71,91,123],"-":[72],"correct":[73],"measures,":[75],"data":[76],"quality":[77],"indicators,":[78],"guardrails":[79],"that":[80],"prevent":[81],"harm":[82],"business,":[84],"comprehensive":[87],"supporting":[90],"understand":[93],"\"why\"":[95],"behind":[96],"movements":[99],"#1":[102],"challenge":[103],"practitioners":[104],"face":[105],"when":[106],"trying":[107,147],"scale":[109],"their":[110],"experimentation":[111],"program":[112],"[18,":[113],"22].":[114],"On":[115],"technical":[117],"side,":[118],"improving":[119],"sensitivity":[120],"hard":[126],"problem":[127],"an":[129],"active":[130],"research":[131,184],"area,":[132],"with":[133],"large":[134],"practical":[135,178],"implications":[136],"as":[137,180,182],"small":[141],"medium":[143],"size":[144],"businesses":[145],"are":[146],"adopt":[149],"suffer":[153],"insufficient":[155],"power.":[156],"In":[157],"this":[158],"tutorial":[159],"we":[160],"will":[161],"discuss":[162],"challenges,":[163],"best":[164],"practices,":[165],"pitfalls":[167],"evaluating":[169],"results,":[171],"focusing":[172],"both":[174],"lessons":[175],"learned":[176],"guidelines":[179],"well":[181],"open":[183],"questions.":[185]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}