{"id":"https://openalex.org/W2127052912","doi":"https://doi.org/10.1145/2566486.2568028","title":"Statistical inference in two-stage online controlled experiments with treatment selection and validation","display_name":"Statistical inference in two-stage online controlled experiments with treatment selection and validation","publication_year":2014,"publication_date":"2014-04-07","ids":{"openalex":"https://openalex.org/W2127052912","doi":"https://doi.org/10.1145/2566486.2568028","mag":"2127052912"},"language":"en","primary_location":{"id":"doi:10.1145/2566486.2568028","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2566486.2568028","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd international conference on World wide web","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075519075","display_name":"Alex Deng","orcid":"https://orcid.org/0000-0002-8116-5602"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Alex Deng","raw_affiliation_strings":["Microsoft, Redmond, WA, USA","Microsoft Redmond, WA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft Redmond, WA, USA#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077899199","display_name":"Tianxi Li","orcid":"https://orcid.org/0000-0003-4595-1777"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan\u2013Ann Arbor","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tianxi Li","raw_affiliation_strings":["University of Michigan, Ann Arbor, MI, USA"],"affiliations":[{"raw_affiliation_string":"University of Michigan, Ann Arbor, MI, USA","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101954335","display_name":"Guo Yu","orcid":"https://orcid.org/0000-0002-3480-6921"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yu Guo","raw_affiliation_strings":["Microsoft, Redmond, WA, USA","Microsoft Redmond, WA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft Redmond, WA, USA#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5075519075"],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":5.3947,"has_fulltext":false,"cited_by_count":27,"citation_normalized_percentile":{"value":0.96078847,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"609","last_page":"618"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11798","display_name":"Optimal Experimental Design Methods","score":0.9833999872207642,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11663","display_name":"Viral Infectious Diseases and Gene Expression in Insects","score":0.9818000197410583,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7085096836090088},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5625576972961426},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.5596652030944824},{"id":"https://openalex.org/keywords/multiple-comparisons-problem","display_name":"Multiple comparisons problem","score":0.554222822189331},{"id":"https://openalex.org/keywords/false-positive-paradox","display_name":"False positive paradox","score":0.5300267338752747},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.5274627208709717},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5147354602813721},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.5017027854919434},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4867582321166992},{"id":"https://openalex.org/keywords/stage","display_name":"Stage (stratigraphy)","score":0.47436317801475525},{"id":"https://openalex.org/keywords/design-of-experiments","display_name":"Design of experiments","score":0.45764583349227905},{"id":"https://openalex.org/keywords/false-positives-and-false-negatives","display_name":"False positives and false negatives","score":0.43679189682006836},{"id":"https://openalex.org/keywords/statistical-process-control","display_name":"Statistical process control","score":0.43584007024765015},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.39036598801612854},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.2440687119960785},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1135251522064209}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7085096836090088},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5625576972961426},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.5596652030944824},{"id":"https://openalex.org/C183905921","wikidata":"https://www.wikidata.org/wiki/Q1038757","display_name":"Multiple comparisons problem","level":2,"score":0.554222822189331},{"id":"https://openalex.org/C64869954","wikidata":"https://www.wikidata.org/wiki/Q1859747","display_name":"False positive paradox","level":2,"score":0.5300267338752747},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5274627208709717},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5147354602813721},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.5017027854919434},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4867582321166992},{"id":"https://openalex.org/C146357865","wikidata":"https://www.wikidata.org/wiki/Q1123245","display_name":"Stage (stratigraphy)","level":2,"score":0.47436317801475525},{"id":"https://openalex.org/C34559072","wikidata":"https://www.wikidata.org/wiki/Q2334061","display_name":"Design of experiments","level":2,"score":0.45764583349227905},{"id":"https://openalex.org/C112789634","wikidata":"https://www.wikidata.org/wiki/Q18207010","display_name":"False positives and false negatives","level":3,"score":0.43679189682006836},{"id":"https://openalex.org/C113644684","wikidata":"https://www.wikidata.org/wiki/Q1356717","display_name":"Statistical process control","level":3,"score":0.43584007024765015},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39036598801612854},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2440687119960785},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1135251522064209},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2566486.2568028","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2566486.2568028","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd international conference on World wide web","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.643.5865","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.643.5865","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.exp-platform.com/Documents/p609-deng.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.6399999856948853,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W363231474","https://openalex.org/W620038288","https://openalex.org/W1510659740","https://openalex.org/W1584444527","https://openalex.org/W1598266570","https://openalex.org/W1993889364","https://openalex.org/W2015810892","https://openalex.org/W2045638068","https://openalex.org/W2062989416","https://openalex.org/W2064039105","https://openalex.org/W2072857774","https://openalex.org/W2074466695","https://openalex.org/W2110065044","https://openalex.org/W2110228583","https://openalex.org/W2111971073","https://openalex.org/W2112508839","https://openalex.org/W2121044470","https://openalex.org/W2126002144","https://openalex.org/W2163996478","https://openalex.org/W2168110243","https://openalex.org/W2169113736","https://openalex.org/W2170051345","https://openalex.org/W2314407721","https://openalex.org/W3215037115","https://openalex.org/W4205110562","https://openalex.org/W4234525067","https://openalex.org/W4251742697","https://openalex.org/W6634927326"],"related_works":["https://openalex.org/W1557094818","https://openalex.org/W2183246718","https://openalex.org/W1973412793","https://openalex.org/W2903672469","https://openalex.org/W2020413892","https://openalex.org/W1991225657","https://openalex.org/W2093463665","https://openalex.org/W2102122784","https://openalex.org/W256699647","https://openalex.org/W1825621501"],"abstract_inverted_index":{"Online":[0],"controlled":[1,209],"experiments,":[2],"also":[3],"called":[4],"A/B":[5,21],"testing,":[6],"have":[7,79],"been":[8],"established":[9,52],"as":[10],"the":[11,94,106,125,129,136,141,146,149,164,177,198],"mantra":[12],"for":[13,38,71,175,187],"data-driven":[14],"decision":[15,24,70],"making":[16,25],"in":[17,101,121,148],"many":[18],"web-facing":[19],"companies.":[20],"Testing":[22],"support":[23],"by":[26],"directly":[27],"comparing":[28],"two":[29,42,80,113,116],"variants":[30],"at":[31],"a":[32,47,66,115,172],"time.":[33],"It":[34],"can":[35],"be":[36],"used":[37],"comparison":[39],"between":[40],"(1)":[41],"candidate":[43,48],"treatments":[44,63,85],"and":[45,50,133,145,192],"(2)":[46],"treatment":[49,127,144,199],"an":[51,59,98],"control.":[53],"In":[54,167],"practice,":[55],"one":[56],"typically":[57],"runs":[58],"experiment":[60,138],"with":[61,65,139,183],"multiple":[62,84,91],"together":[64,182],"control":[67,147],"to":[68,78,90,206],"make":[69],"both":[72],"purposes":[73],"simultaneously.":[74],"This":[75],"is":[76,119,203],"known":[77],"issues.":[81],"First,":[82],"having":[83],"increases":[86],"false":[87],"positives":[88],"due":[89],"comparison.":[92],"Second,":[93],"selection":[95],"process":[96,118],"causes":[97],"upward":[99],"bias":[100],"estimated":[102],"effect":[103],"size":[104],"of":[105,154,197],"best":[107,126,143],"observed":[108],"treatment.":[109],"To":[110],"overcome":[111],"these":[112],"issues,":[114],"stage":[117,132,180,185],"recommended,":[120],"which":[122],"we":[123,170],"select":[124],"from":[128,163],"first":[130,178],"screening":[131,179],"then":[134],"run":[135],"same":[137],"only":[140,160],"selected":[142],"validation":[150,184],"stage.":[151,166],"Traditional":[152],"application":[153],"this":[155,168],"two-stage":[156],"design":[157],"often":[158],"focus":[159],"on":[161],"results":[162],"second":[165],"paper,":[169],"propose":[171],"general":[173],"methodology":[174],"combining":[176],"data":[181,186],"more":[188,193],"sensitive":[189],"hypothesis":[190],"testing":[191],"accurate":[194],"point":[195],"estimation":[196],"effect.":[200],"Our":[201],"method":[202],"widely":[204],"applicable":[205],"existing":[207],"online":[208],"experimentation":[210],"systems.":[211]},"counts_by_year":[{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":5},{"year":2016,"cited_by_count":4},{"year":2015,"cited_by_count":7}],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2025-10-10T00:00:00"}