{"id":"https://openalex.org/W2744538883","doi":"https://doi.org/10.1145/3097983.3097992","title":"Peeking at A/B Tests","display_name":"Peeking at A/B Tests","publication_year":2017,"publication_date":"2017-08-04","ids":{"openalex":"https://openalex.org/W2744538883","doi":"https://doi.org/10.1145/3097983.3097992","mag":"2744538883"},"language":"en","primary_location":{"id":"doi:10.1145/3097983.3097992","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3097983.3097992","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042177635","display_name":"Ramesh Johari","orcid":"https://orcid.org/0000-0002-3960-0770"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ramesh Johari","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022151692","display_name":"Pete Koomen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pete Koomen","raw_affiliation_strings":["Optimizely, Inc., San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Optimizely, Inc., San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013984696","display_name":"Leonid Pekelis","orcid":"https://orcid.org/0000-0002-3126-9278"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Leonid Pekelis","raw_affiliation_strings":["Optimizely, Inc., San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Optimizely, Inc., San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032726305","display_name":"David Walsh","orcid":"https://orcid.org/0000-0003-2568-1802"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Walsh","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5042177635"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":17.7363,"has_fulltext":false,"cited_by_count":129,"citation_normalized_percentile":{"value":0.9961587,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1517","last_page":"1525"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11798","display_name":"Optimal Experimental Design Methods","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7037250399589539},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6284853219985962},{"id":"https://openalex.org/keywords/confidence-interval","display_name":"Confidence interval","score":0.5953776836395264},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5620037317276001},{"id":"https://openalex.org/keywords/statistical-inference","display_name":"Statistical inference","score":0.5195610523223877},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.5145747065544128},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.4650344252586365},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3954116106033325},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.37003687024116516},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.26682063937187195},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.22433346509933472},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.16234827041625977},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.12363451719284058},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09874221682548523}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7037250399589539},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6284853219985962},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.5953776836395264},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5620037317276001},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.5195610523223877},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.5145747065544128},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.4650344252586365},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3954116106033325},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37003687024116516},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26682063937187195},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.22433346509933472},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.16234827041625977},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.12363451719284058},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09874221682548523}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3097983.3097992","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3097983.3097992","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W39972009","https://openalex.org/W1584444527","https://openalex.org/W1596515083","https://openalex.org/W1970385173","https://openalex.org/W1975566260","https://openalex.org/W1982183183","https://openalex.org/W2028125409","https://openalex.org/W2028785275","https://openalex.org/W2039119502","https://openalex.org/W2061554433","https://openalex.org/W2069523442","https://openalex.org/W2073997076","https://openalex.org/W2091082553","https://openalex.org/W2110065044","https://openalex.org/W2112508839","https://openalex.org/W2130527179","https://openalex.org/W2167770337","https://openalex.org/W2244314045","https://openalex.org/W2293233856","https://openalex.org/W2323136350","https://openalex.org/W2397891505","https://openalex.org/W4230704549","https://openalex.org/W4235402124","https://openalex.org/W4256232412","https://openalex.org/W4299922653"],"related_works":["https://openalex.org/W137830373","https://openalex.org/W3000984192","https://openalex.org/W2103073163","https://openalex.org/W4286952477","https://openalex.org/W2461683294","https://openalex.org/W4321348134","https://openalex.org/W4387929287","https://openalex.org/W4226211120","https://openalex.org/W1987205174","https://openalex.org/W2380849986"],"abstract_inverted_index":{"This":[0,39,97],"paper":[1,98],"reports":[2],"on":[3,104],"novel":[4],"statistical":[5],"methodology,":[6],"which":[7],"has":[8],"been":[9],"deployed":[10],"by":[11],"the":[12,28,55],"commercial":[13],"A/B":[14,44],"testing":[15,45],"platform":[16],"Optimizely":[17],"to":[18,22,49,71,84,91],"communicate":[19],"experimental":[20],"results":[21],"their":[23],"customers.":[24],"Our":[25],"methodology":[26],"addresses":[27],"issue":[29],"that":[30,67],"traditional":[31,114],"p-values":[32,63],"and":[33,64,101],"confidence":[34,65],"intervals":[35,66],"give":[36],"unreliable":[37],"inference.":[38],"is":[40,57],"because":[41],"users":[42],"of":[43],"software":[46],"are":[47,68],"known":[48],"continuously":[50,85],"monitor":[51],"these":[52],"measures":[53],"as":[54],"experiment":[56],"running.":[58],"We":[59],"provide":[60],"always":[61],"valid":[62],"provably":[69],"robust":[70],"this":[72,77],"effect.":[73],"Not":[74],"only":[75],"does":[76],"make":[78],"it":[79,88],"safe":[80],"for":[81],"a":[82],"user":[83],"monitor,":[86],"but":[87],"empowers":[89],"her":[90],"detect":[92],"true":[93],"effects":[94],"more":[95],"efficiently.":[96],"provides":[99],"simulations":[100],"numerical":[102],"studies":[103],"Optimizely's":[105],"data,":[106],"demonstrating":[107],"an":[108],"improvement":[109],"in":[110],"detection":[111],"performance":[112],"over":[113],"methods.":[115]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":13},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":18},{"year":2020,"cited_by_count":51},{"year":2019,"cited_by_count":13},{"year":2018,"cited_by_count":9}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}