{"id":"https://openalex.org/W4288480961","doi":"https://doi.org/10.1145/3303772.3303812","title":"Beyond A/B Testing","display_name":"Beyond A/B Testing","publication_year":2019,"publication_date":"2019-02-25","ids":{"openalex":"https://openalex.org/W4288480961","doi":"https://doi.org/10.1145/3303772.3303812"},"language":"en","primary_location":{"id":"doi:10.1145/3303772.3303812","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3303772.3303812","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 9th International Conference on Learning Analytics & Knowledge","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048161983","display_name":"Timothy NeCamp","orcid":"https://orcid.org/0000-0002-0192-6230"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan\u2013Ann Arbor","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Timothy NeCamp","raw_affiliation_strings":["Department of Statistics, University of Michigan, Ann Arbor, Michigan, USA"],"affiliations":[{"raw_affiliation_string":"Department of Statistics, University of Michigan, Ann Arbor, Michigan, USA","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110941464","display_name":"Josh Gardner","orcid":null},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Josh Gardner","raw_affiliation_strings":["Paul G. Allen School of Computer Science & Engineering, University of Washington, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081802677","display_name":"Christopher Brooks","orcid":"https://orcid.org/0000-0003-0875-0204"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan\u2013Ann Arbor","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christopher Brooks","raw_affiliation_strings":["School of Information, University of Michigan, Ann Arbor, Michigan, USA"],"affiliations":[{"raw_affiliation_string":"School of Information, University of Michigan, Ann Arbor, Michigan, USA","institution_ids":["https://openalex.org/I27837315"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5048161983"],"corresponding_institution_ids":["https://openalex.org/I27837315"],"apc_list":null,"apc_paid":null,"fwci":1.92350424,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.84590546,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"539","last_page":"548"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9830999970436096,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9635000228881836,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/psychological-intervention","display_name":"Psychological intervention","score":0.7264596819877625},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6980235576629639},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.514672577381134},{"id":"https://openalex.org/keywords/causal-inference","display_name":"Causal inference","score":0.4979989528656006},{"id":"https://openalex.org/keywords/randomized-experiment","display_name":"Randomized experiment","score":0.4770749807357788},{"id":"https://openalex.org/keywords/randomized-controlled-trial","display_name":"Randomized controlled trial","score":0.4726262092590332},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.47072163224220276},{"id":"https://openalex.org/keywords/analytics","display_name":"Analytics","score":0.4606262743473053},{"id":"https://openalex.org/keywords/learning-analytics","display_name":"Learning analytics","score":0.4488822817802429},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.44546476006507874},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.23258784413337708},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.21241942048072815},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.11434748768806458}],"concepts":[{"id":"https://openalex.org/C27415008","wikidata":"https://www.wikidata.org/wiki/Q7256382","display_name":"Psychological intervention","level":2,"score":0.7264596819877625},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6980235576629639},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.514672577381134},{"id":"https://openalex.org/C158600405","wikidata":"https://www.wikidata.org/wiki/Q5054566","display_name":"Causal inference","level":2,"score":0.4979989528656006},{"id":"https://openalex.org/C155108698","wikidata":"https://www.wikidata.org/wiki/Q1231081","display_name":"Randomized experiment","level":2,"score":0.4770749807357788},{"id":"https://openalex.org/C168563851","wikidata":"https://www.wikidata.org/wiki/Q1436668","display_name":"Randomized controlled trial","level":2,"score":0.4726262092590332},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.47072163224220276},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.4606262743473053},{"id":"https://openalex.org/C2777648619","wikidata":"https://www.wikidata.org/wiki/Q2845208","display_name":"Learning analytics","level":2,"score":0.4488822817802429},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.44546476006507874},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.23258784413337708},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21241942048072815},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.11434748768806458},{"id":"https://openalex.org/C118552586","wikidata":"https://www.wikidata.org/wiki/Q7867","display_name":"Psychiatry","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C141071460","wikidata":"https://www.wikidata.org/wiki/Q40821","display_name":"Surgery","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3303772.3303812","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3303772.3303812","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 9th International Conference on Learning Analytics & Knowledge","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W1184128069","https://openalex.org/W1508428170","https://openalex.org/W1510413338","https://openalex.org/W1522024619","https://openalex.org/W1754268392","https://openalex.org/W1896083716","https://openalex.org/W1965237197","https://openalex.org/W1986546901","https://openalex.org/W1991676478","https://openalex.org/W2007151269","https://openalex.org/W2012265254","https://openalex.org/W2027795129","https://openalex.org/W2039021895","https://openalex.org/W2044439361","https://openalex.org/W2069897595","https://openalex.org/W2118940666","https://openalex.org/W2129707978","https://openalex.org/W2160889022","https://openalex.org/W2163084572","https://openalex.org/W2276801646","https://openalex.org/W2339576062","https://openalex.org/W2341021244","https://openalex.org/W2474213459","https://openalex.org/W2515454359","https://openalex.org/W2549443287","https://openalex.org/W2577350134","https://openalex.org/W2606188057","https://openalex.org/W2610200263","https://openalex.org/W2744079905","https://openalex.org/W2769019609","https://openalex.org/W2769718706","https://openalex.org/W2789942425","https://openalex.org/W2795381838","https://openalex.org/W2797715130","https://openalex.org/W2809633797","https://openalex.org/W2887661611","https://openalex.org/W2887764622","https://openalex.org/W2899145913","https://openalex.org/W2944637756","https://openalex.org/W2992939092","https://openalex.org/W3102460569","https://openalex.org/W3124089888","https://openalex.org/W4301337144","https://openalex.org/W4301344865","https://openalex.org/W4378898217"],"related_works":["https://openalex.org/W4389471064","https://openalex.org/W2119346805","https://openalex.org/W1987827786","https://openalex.org/W2799586942","https://openalex.org/W4322726883","https://openalex.org/W2504091800","https://openalex.org/W2331775400","https://openalex.org/W2816728186","https://openalex.org/W2768832457","https://openalex.org/W2804624249"],"abstract_inverted_index":{"Randomized":[0],"experiments":[1,33],"ensure":[2],"robust":[3],"causal":[4],"inference":[5],"that":[6,89],"are":[7,23,41],"critical":[8],"to":[9,44,47,50,99,122,163,168],"effective":[10],"learning":[11,29,71,175],"analytics":[12],"research":[13],"and":[14,86,102,114],"practice.":[15],"However,":[16],"traditional":[17,32,123],"randomized":[18,76],"experiments,":[19],"like":[20],"A/B":[21],"tests,":[22],"limiting":[24],"in":[25,68,131,172],"large":[26,133],"scale":[27,134],"digital":[28,70,174],"environments.":[30,176],"While":[31],"can":[34,145],"accurately":[35],"compare":[36],"two":[37],"treatment":[38],"options,":[39],"they":[40,119],"less":[42],"able":[43],"inform":[45,97],"how":[46,98,142],"adapt":[48],"interventions":[49,67,88,171],"continually":[51],"meet":[52],"learners'":[53],"diverse":[54],"needs.":[55],"In":[56,105],"this":[57,106],"work,":[58],"we":[59,108,115],"introduce":[60],"a":[61,127,132],"trial":[62,77,139],"design":[63],"for":[64,159],"developing":[65,87],"adaptive":[66,170],"scaled":[69,173],"environments":[72],"--":[73],"the":[74,80,117],"sequential":[75],"(SRT).":[78],"With":[79],"goal":[81],"of":[82,112],"improving":[83],"learner":[84,143],"experience":[85],"benefit":[90],"all":[91,94],"learners":[92],"at":[93],"times,":[95],"SRTs":[96,167],"sequence,":[100],"time,":[101],"personalize":[103],"interventions.":[104],"paper,":[107],"provide":[109,156],"an":[110],"overview":[111],"SRTs,":[113],"illustrate":[116],"advantages":[118],"hold":[120],"compared":[121],"experiments.":[124],"We":[125,154],"describe":[126],"novel":[128],"SRT":[129],"run":[130,164],"data":[135],"science":[136],"MOOC.":[137],"The":[138],"results":[140],"contextualize":[141],"engagement":[144],"be":[146],"addressed":[147],"through":[148],"inclusive":[149],"culturally":[150],"targeted":[151],"reminder":[152],"emails.":[153],"also":[155],"practical":[157],"advice":[158],"researchers":[160],"who":[161],"aim":[162],"their":[165],"own":[166],"develop":[169]},"counts_by_year":[{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":1}],"updated_date":"2025-11-25T21:42:39.735039","created_date":"2022-07-29T00:00:00"}