{"id":"https://openalex.org/W2965478572","doi":"https://doi.org/10.1145/3292500.3332282","title":"Foundations of Large-Scale Sequential Experimentation","display_name":"Foundations of Large-Scale Sequential Experimentation","publication_year":2019,"publication_date":"2019-07-25","ids":{"openalex":"https://openalex.org/W2965478572","doi":"https://doi.org/10.1145/3292500.3332282","mag":"2965478572"},"language":"en","primary_location":{"id":"doi:10.1145/3292500.3332282","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3292500.3332282","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032389695","display_name":"Aaditya Ramdas","orcid":"https://orcid.org/0000-0003-0497-311X"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Aaditya Ramdas","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5032389695"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":0.2341,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.56948132,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"35","issue":null,"first_page":"3211","last_page":"3212"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11443","display_name":"Advanced Statistical Process Monitoring","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11798","display_name":"Optimal Experimental Design Methods","score":0.9728999733924866,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sequential-analysis","display_name":"Sequential analysis","score":0.7052353620529175},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6977487206459045},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.5653130412101746},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5579574704170227},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5429626703262329},{"id":"https://openalex.org/keywords/regret","display_name":"Regret","score":0.47196850180625916},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4697224199771881},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.43897518515586853},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.43085533380508423},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3539867401123047},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3420882225036621},{"id":"https://openalex.org/keywords/operations-research","display_name":"Operations research","score":0.33602502942085266},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1444219946861267},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.11607727408409119}],"concepts":[{"id":"https://openalex.org/C80478641","wikidata":"https://www.wikidata.org/wiki/Q195771","display_name":"Sequential analysis","level":2,"score":0.7052353620529175},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6977487206459045},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.5653130412101746},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5579574704170227},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5429626703262329},{"id":"https://openalex.org/C50817715","wikidata":"https://www.wikidata.org/wiki/Q79895177","display_name":"Regret","level":2,"score":0.47196850180625916},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4697224199771881},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.43897518515586853},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.43085533380508423},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3539867401123047},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3420882225036621},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.33602502942085266},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1444219946861267},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.11607727408409119},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3292500.3332282","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3292500.3332282","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5799999833106995,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W1506859583","https://openalex.org/W1995692164","https://openalex.org/W2019506501","https://openalex.org/W2047023085","https://openalex.org/W2110065044","https://openalex.org/W2167770337","https://openalex.org/W2552366573","https://openalex.org/W2744538883","https://openalex.org/W2761626342","https://openalex.org/W2913668833","https://openalex.org/W2962900746","https://openalex.org/W2963562682","https://openalex.org/W2963651028","https://openalex.org/W2963736810","https://openalex.org/W2963790862","https://openalex.org/W3003196782","https://openalex.org/W4230704549","https://openalex.org/W4232896127"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4224009465","https://openalex.org/W2984358193","https://openalex.org/W4286629047","https://openalex.org/W4306321456","https://openalex.org/W4285260836","https://openalex.org/W3046775127","https://openalex.org/W3170094116","https://openalex.org/W4205958290"],"abstract_inverted_index":{"Large-scale":[0],"sequential":[1,34,57,61,67,218],"hypothesis":[2],"testing":[3,125,162],"(A/B-testing)":[4],"is":[5,24,183],"rampant":[6],"in":[7,48,197,233],"the":[8,39,44,55,65,147,155,170,198,213,216,223,234,238],"tech":[9,199],"industry,":[10],"with":[11,103,215],"internet":[12,190],"companies":[13],"running":[14],"hundreds":[15],"of":[16,18,30,33,71,121,157,225,237],"thousands":[17],"tests":[19],"per":[20],"year.":[21],"This":[22,181],"experimentation":[23,148],"actually":[25],"\"doubly-sequential\",":[26],"since":[27],"it":[28],"consists":[29],"a":[31,207],"sequence":[32,70],"experiments.":[35],"In":[36,112],"this":[37],"tutorial,":[38],"audience":[40],"will":[41,84,115,151,230],"learn":[42,74],"about":[43,75],"various":[45],"problems":[46],"encountered":[47],"large-scale,":[49],"asynchronous,":[50],"doubly-sequential":[51],"experimentation,":[52],"both":[53,88,102,130],"for":[54,64,161],"inner":[56],"process":[58,68,219],"(a":[59],"single":[60,189,208],"test)":[62],"and":[63,73,90,93,104,123,132,169],"outer":[66,217],"(the":[69],"tests),":[72],"recently":[76],"developed":[77],"principles":[78],"to":[79,107,127,146,166,174,205],"tackle":[80],"these":[81,100,137],"problems.":[82],"We":[83,150],"discuss":[85,154],"error":[86],"metrics":[87],"within":[89,131],"across":[91,133],"experiments,":[92,134],"present":[94],"state-of-the-art":[95],"methods":[96,160],"that":[97],"provably":[98],"control":[99,128],"errors,":[101],"without":[105],"resorting":[106],"parametric":[108],"or":[109],"asymptotic":[110],"assumptions.":[111],"particular,":[113],"we":[114,229],"demonstrate":[116],"how":[117,136,204],"current":[118],"common":[119],"practices":[120],"peeking":[122],"marginal":[124],"fail":[126],"errors":[129],"but":[135],"can":[138],"be":[139],"alleviated":[140],"using":[141],"simple":[142],"yet":[143],"nuanced":[144],"changes":[145],"setup.":[149],"also":[152],"briefly":[153],"role":[156],"multi-armed":[158],"bandit":[159],"hypotheses":[163],"(as":[164],"opposed":[165],"minimizing":[167],"regret),":[168],"potential":[171],"pitfalls":[172],"due":[173],"selection":[175],"bias":[176],"introduced":[177],"by":[178],"adaptive":[179],"sampling.":[180],"tutorial":[182],"timely":[184],"because":[185],"while":[186],"almost":[187],"every":[188],"company":[191],"runs":[192],"such":[193],"tests,":[194],"most":[195],"practitioners":[196],"industry":[200],"focus":[201],"mainly":[202],"on":[203],"run":[206],"test":[209],"correctly.":[210],"However,":[211],"ignoring":[212],"interplay":[214],"could":[220],"unknowingly":[221],"inflate":[222],"number":[224],"false":[226],"discoveries,":[227],"as":[228],"carefully":[231],"explain":[232],"second":[235],"half":[236],"tutorial.":[239]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}