{"id":"https://openalex.org/W4321471897","doi":"https://doi.org/10.1145/3543873.3584635","title":"Anytime-Valid Confidence Sequences in an Enterprise A/B Testing Platform","display_name":"Anytime-Valid Confidence Sequences in an Enterprise A/B Testing Platform","publication_year":2023,"publication_date":"2023-04-28","ids":{"openalex":"https://openalex.org/W4321471897","doi":"https://doi.org/10.1145/3543873.3584635"},"language":"en","primary_location":{"id":"doi:10.1145/3543873.3584635","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3543873.3584635","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3543873.3584635","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the ACM Web Conference 2023","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3543873.3584635","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091474850","display_name":"Akash V. Maharaj","orcid":"https://orcid.org/0000-0001-8625-1844"},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Akash Maharaj","raw_affiliation_strings":["Adobe, USA"],"raw_orcid":"https://orcid.org/0000-0001-8625-1844","affiliations":[{"raw_affiliation_string":"Adobe, USA","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056964468","display_name":"Ritwik Sinha","orcid":"https://orcid.org/0000-0002-0344-1368"},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ritwik Sinha","raw_affiliation_strings":["Adobe Research, USA"],"raw_orcid":"https://orcid.org/0000-0002-0344-1368","affiliations":[{"raw_affiliation_string":"Adobe Research, USA","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063845534","display_name":"David Arbour","orcid":"https://orcid.org/0000-0002-9932-7657"},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Arbour","raw_affiliation_strings":["Adobe Research, USA"],"raw_orcid":"https://orcid.org/0000-0002-9932-7657","affiliations":[{"raw_affiliation_string":"Adobe Research, USA","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048983595","display_name":"Ian Waudby-Smith","orcid":"https://orcid.org/0000-0003-4471-1405"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ian Waudby-Smith","raw_affiliation_strings":["Carnegie Mellon University, USA"],"raw_orcid":"https://orcid.org/0000-0003-4471-1405","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000443169","display_name":"Simon Z. Liu","orcid":"https://orcid.org/0000-0002-6002-742X"},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Simon Z. Liu","raw_affiliation_strings":["Adobe, USA"],"raw_orcid":"https://orcid.org/0000-0002-6002-742X","affiliations":[{"raw_affiliation_string":"Adobe, USA","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102835950","display_name":"Moumita Sinha","orcid":"https://orcid.org/0000-0001-5634-4345"},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Moumita Sinha","raw_affiliation_strings":["Adobe, USA"],"raw_orcid":"https://orcid.org/0000-0001-5634-4345","affiliations":[{"raw_affiliation_string":"Adobe, USA","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025800378","display_name":"Raghavendra Addanki","orcid":"https://orcid.org/0000-0003-1478-083X"},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Raghavendra Addanki","raw_affiliation_strings":["Adobe Research, USA"],"raw_orcid":"https://orcid.org/0000-0003-1478-083X","affiliations":[{"raw_affiliation_string":"Adobe Research, USA","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032389695","display_name":"Aaditya Ramdas","orcid":"https://orcid.org/0000-0003-0497-311X"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aaditya Ramdas","raw_affiliation_strings":["Carnegie Mellon University, USA"],"raw_orcid":"https://orcid.org/0000-0003-0497-311X","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003864929","display_name":"M Garg","orcid":"https://orcid.org/0000-0002-1060-8939"},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Manas Garg","raw_affiliation_strings":["Adobe, USA"],"raw_orcid":"https://orcid.org/0000-0002-1060-8939","affiliations":[{"raw_affiliation_string":"Adobe, USA","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5022934605","display_name":"Viswanathan Swaminathan","orcid":"https://orcid.org/0000-0002-1357-1502"},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Viswanathan Swaminathan","raw_affiliation_strings":["Adobe Research, USA"],"raw_orcid":"https://orcid.org/0000-0002-1357-1502","affiliations":[{"raw_affiliation_string":"Adobe Research, USA","institution_ids":["https://openalex.org/I1306409833"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.666,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.96701847,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"396","last_page":"400"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11443","display_name":"Advanced Statistical Process Monitoring","score":0.9757000207901001,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11195","display_name":"Simulation Techniques and Applications","score":0.9747999906539917,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6165778636932373},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3610551655292511}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6165778636932373},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3610551655292511}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3543873.3584635","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3543873.3584635","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3543873.3584635","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the ACM Web Conference 2023","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2302.10108","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2302.10108","pdf_url":"https://arxiv.org/pdf/2302.10108","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3543873.3584635","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3543873.3584635","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3543873.3584635","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the ACM Web Conference 2023","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.6000000238418579,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4321471897.pdf","grobid_xml":"https://content.openalex.org/works/W4321471897.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W1987820735","https://openalex.org/W1990639292","https://openalex.org/W2031257157","https://openalex.org/W2039119502","https://openalex.org/W2054779505","https://openalex.org/W2059707463","https://openalex.org/W2060499368","https://openalex.org/W2104235511","https://openalex.org/W2127151679","https://openalex.org/W2149378760","https://openalex.org/W2744538883","https://openalex.org/W2912599363","https://openalex.org/W2963366444","https://openalex.org/W3030671780","https://openalex.org/W3034080994","https://openalex.org/W3102572863","https://openalex.org/W3126216366","https://openalex.org/W3148231156","https://openalex.org/W3190343448","https://openalex.org/W4230704549","https://openalex.org/W4242589154","https://openalex.org/W4255434707","https://openalex.org/W4295677904","https://openalex.org/W4380149598","https://openalex.org/W4388461534","https://openalex.org/W4388561664"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"A/B":[0,76,95],"tests":[1,41],"are":[2,18],"the":[3,11,22,61,75,127],"gold":[4],"standard":[5],"for":[6,71],"evaluating":[7],"digital":[8],"experiences":[9],"on":[10,60,66],"web.":[12],"However,":[13],"traditional":[14,136],"\u201cfixed-horizon\u201d":[15],"statistical":[16],"methods":[17,133],"often":[19],"incompatible":[20],"with":[21],"needs":[23],"of":[24,35,39,74,122,130,135],"modern":[25],"industry":[26],"practitioners":[27],"as":[28],"they":[29],"do":[30],"not":[31],"permit":[32],"continuous":[33,72],"monitoring":[34,73],"experiments.":[36],"Frequent":[37],"evaluation":[38],"fixed-horizon":[40],"(\u201cpeeking\u201d)":[42],"leads":[43],"to":[44],"inflated":[45],"type-I":[46],"error":[47],"and":[48,78,86,106,120],"can":[49,103,113],"result":[50],"in":[51,91],"erroneous":[52],"conclusions.":[53],"We":[54,81],"have":[55],"released":[56],"an":[57],"experimentation":[58],"service":[59],"Adobe":[62],"Experience":[63],"Platform":[64],"based":[65],"anytime-valid":[67,132],"confidence":[68,89],"sequences,":[69],"allowing":[70],"test":[77,109],"data-dependent":[79],"stopping.":[80],"demonstrate":[82],"how":[83,99,107],"we":[84,125],"adapted":[85],"deployed":[87],"asymptotic":[88],"sequences":[90],"a":[92],"full":[93],"featured":[94],"testing":[96],"platform,":[97],"describe":[98],"sample":[100],"size":[101],"calculations":[102],"be":[104,114],"performed,":[105],"alternate":[108],"statistics":[110],"like":[111],"\u201clift\u201d":[112],"analyzed.":[115],"On":[116],"both":[117],"simulated":[118],"data":[119],"thousands":[121],"real":[123],"experiments,":[124],"show":[126],"desirable":[128],"properties":[129],"using":[131],"instead":[134],"approaches.":[137]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-31T08:31:51.225901","created_date":"2025-10-10T00:00:00"}