{"id":"https://openalex.org/W3020904298","doi":"https://doi.org/10.1145/3366424.3383117","title":"Challenges, Best Practices and Pitfalls in Evaluating Results of Online Controlled Experiments","display_name":"Challenges, Best Practices and Pitfalls in Evaluating Results of Online Controlled Experiments","publication_year":2020,"publication_date":"2020-04-20","ids":{"openalex":"https://openalex.org/W3020904298","doi":"https://doi.org/10.1145/3366424.3383117","mag":"3020904298"},"language":"en","primary_location":{"id":"doi:10.1145/3366424.3383117","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366424.3383117","pdf_url":null,"source":{"id":"https://openalex.org/S4306506651","display_name":"Companion Proceedings of the Web Conference 2020","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the Web Conference 2020","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3366424.3383117","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039548707","display_name":"Somit Gupta","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Somit Gupta","raw_affiliation_strings":["Microsoft Redmond, WA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101666398","display_name":"Xiaolin Shi","orcid":"https://orcid.org/0000-0002-3705-1552"},"institutions":[{"id":"https://openalex.org/I4210142583","display_name":"Snap (United States)","ror":"https://ror.org/04dgkhg68","country_code":"US","type":"company","lineage":["https://openalex.org/I4210142583"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaolin Shi","raw_affiliation_strings":["Snap Inc. Santa Monica, CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Snap Inc. Santa Monica, CA","institution_ids":["https://openalex.org/I4210142583"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020697231","display_name":"Pavel Dmitriev","orcid":"https://orcid.org/0000-0001-5740-5146"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pavel Dmitriev","raw_affiliation_strings":["Outreach Seattle, WA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Outreach Seattle, WA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068016934","display_name":"Xin Fu","orcid":"https://orcid.org/0000-0003-4015-0778"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xin Fu","raw_affiliation_strings":["Facebook Menlo Park, CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook Menlo Park, CA","institution_ids":["https://openalex.org/I4210114444","https://openalex.org/I4210099336"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"317","last_page":"319"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9175000190734863,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9175000190734863,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10791","display_name":"Advanced Control Systems Optimization","score":0.902999997138977,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6499772667884827},{"id":"https://openalex.org/keywords/best-practice","display_name":"Best practice","score":0.432953804731369},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.39487960934638977},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.0665678083896637}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6499772667884827},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.432953804731369},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.39487960934638977},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0665678083896637},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3366424.3383117","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366424.3383117","pdf_url":null,"source":{"id":"https://openalex.org/S4306506651","display_name":"Companion Proceedings of the Web Conference 2020","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the Web Conference 2020","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3366424.3383117","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366424.3383117","pdf_url":null,"source":{"id":"https://openalex.org/S4306506651","display_name":"Companion Proceedings of the Web Conference 2020","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the Web Conference 2020","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.5899999737739563,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W2070815407","https://openalex.org/W2114968482","https://openalex.org/W2126002144","https://openalex.org/W2169113736","https://openalex.org/W2170443290","https://openalex.org/W2468478068","https://openalex.org/W2517816274","https://openalex.org/W2584778415","https://openalex.org/W2740607000","https://openalex.org/W2743849295","https://openalex.org/W2748827028","https://openalex.org/W2945720420","https://openalex.org/W2946387282","https://openalex.org/W2952127798","https://openalex.org/W2954602266","https://openalex.org/W2965222965"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2002383399","https://openalex.org/W644644594","https://openalex.org/W2506073049","https://openalex.org/W642180557","https://openalex.org/W2780591772","https://openalex.org/W3122105723","https://openalex.org/W1593090812","https://openalex.org/W2390279801"],"abstract_inverted_index":{"A/B":[0,59,150],"Testing":[1],"is":[2,25,61,100,124],"the":[3,8,29,67,94,97,101,116,212],"gold":[4],"standard":[5],"to":[6,31,42,51,83,92,108,148],"estimate":[7],"causal":[9],"relationship":[10],"between":[11],"a":[12,15,86,125],"change":[13,38,41],"in":[14,28,168],"product":[16],"and":[17,85,128,139,142,152,166,177],"its":[18],"impact":[19],"on":[20,173],"key":[21,56,98],"outcome":[22,74],"measures.":[23],"It":[24,198],"widely":[26],"used":[27],"industry":[30],"test":[32],"changes":[33,45],"ranging":[34],"from":[35,154],"simple":[36],"copy":[37],"or":[39],"UI":[40],"more":[43,138,140],"complex":[44],"like":[46],"using":[47],"machine":[48],"learning":[49],"models":[50],"personalize":[52],"user":[53],"experience.":[54],"The":[55],"aspect":[57],"of":[58,63,70,89,121,188],"testing":[60,151],"evaluation":[62],"experiment":[64,122,170],"results.":[65],"Designing":[66],"right":[68],"set":[69,88],"metrics":[71,91,123],"-":[72],"correct":[73],"measures,":[75],"data":[76],"quality":[77],"indicators,":[78],"guardrails":[79],"that":[80],"prevent":[81],"harm":[82],"business,":[84],"comprehensive":[87],"supporting":[90],"understand":[93],"\u201cwhy\u201d":[95],"behind":[96],"movements":[99],"#1":[102],"challenge":[103],"practitioners":[104],"face":[105],"when":[106],"trying":[107,147],"scale":[109],"their":[110],"experimentation":[111],"program":[112],"11,":[113],"14.":[114],"On":[115],"technical":[117],"side,":[118],"improving":[119],"sensitivity":[120],"hard":[126],"problem":[127],"an":[129],"active":[130],"research":[131,184],"area,":[132],"with":[133],"large":[134],"practical":[135,178],"implications":[136],"as":[137,180,182],"small":[141],"medium":[143],"size":[144],"businesses":[145],"are":[146],"adopt":[149],"suffer":[153],"insufficient":[155],"power.":[156],"In":[157],"this":[158,189],"tutorial":[159,190,206],"we":[160],"will":[161],"discuss":[162],"challenges,":[163],"best":[164],"practices,":[165],"pitfalls":[167],"evaluating":[169],"results,":[171],"focusing":[172],"both":[174],"lessons":[175],"learned":[176],"guidelines":[179],"well":[181],"open":[183],"questions.":[185],"A":[186],"version":[187],"was":[191,199],"also":[192,208],"present":[193],"at":[194],"KDD":[195],"2019":[196],"23.":[197],"attended":[200],"by":[201],"around":[202],"150":[203],"participants.":[204],"This":[205],"has":[207],"been":[209],"accepted":[210],"for":[211],"WSDM":[213],"2020":[214],"conference.":[215]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}