{"id":"https://openalex.org/W2169113736","doi":"https://doi.org/10.1145/2339530.2339653","title":"Trustworthy online controlled experiments","display_name":"Trustworthy online controlled experiments","publication_year":2012,"publication_date":"2012-08-12","ids":{"openalex":"https://openalex.org/W2169113736","doi":"https://doi.org/10.1145/2339530.2339653","mag":"2169113736"},"language":"en","primary_location":{"id":"doi:10.1145/2339530.2339653","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2339530.2339653","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th ACM SIGKDD international conference on Knowledge discovery and data mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037339239","display_name":"Ron Kohavi","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ron Kohavi","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075519075","display_name":"Alex Deng","orcid":"https://orcid.org/0000-0002-8116-5602"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alex Deng","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089128609","display_name":"Brian Frasca","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Brian Frasca","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050159156","display_name":"Roger Longbotham","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Roger Longbotham","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031583581","display_name":"Toby Walker","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Toby Walker","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5114062370","display_name":"Ya Xu","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ya Xu","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5037339239"],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":17.7859,"has_fulltext":false,"cited_by_count":224,"citation_normalized_percentile":{"value":0.99593936,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"786","last_page":"794"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.819599986076355,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.819599986076355,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.8195000290870667,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11875","display_name":"Statistics Education and Methodologies","score":0.807200014591217,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.7038610577583313},{"id":"https://openalex.org/keywords/revenue","display_name":"Revenue","score":0.6328952312469482},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.625868558883667},{"id":"https://openalex.org/keywords/root","display_name":"Root (linguistics)","score":0.5425267219543457},{"id":"https://openalex.org/keywords/stochastic-game","display_name":"Stochastic game","score":0.5328709483146667},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.5102560520172119},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.43158629536628723},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.4195060431957245},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3773535490036011},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.2921738624572754},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.1791989803314209},{"id":"https://openalex.org/keywords/microeconomics","display_name":"Microeconomics","score":0.1575053632259369},{"id":"https://openalex.org/keywords/accounting","display_name":"Accounting","score":0.10206830501556396}],"concepts":[{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.7038610577583313},{"id":"https://openalex.org/C195487862","wikidata":"https://www.wikidata.org/wiki/Q850210","display_name":"Revenue","level":2,"score":0.6328952312469482},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.625868558883667},{"id":"https://openalex.org/C171078966","wikidata":"https://www.wikidata.org/wiki/Q111029","display_name":"Root (linguistics)","level":2,"score":0.5425267219543457},{"id":"https://openalex.org/C22171661","wikidata":"https://www.wikidata.org/wiki/Q1074380","display_name":"Stochastic game","level":2,"score":0.5328709483146667},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.5102560520172119},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.43158629536628723},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.4195060431957245},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3773535490036011},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.2921738624572754},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.1791989803314209},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.1575053632259369},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.10206830501556396},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2339530.2339653","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2339530.2339653","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th ACM SIGKDD international conference on Knowledge discovery and data mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W219315687","https://openalex.org/W1486094596","https://openalex.org/W1553296127","https://openalex.org/W1568732389","https://openalex.org/W1580090600","https://openalex.org/W1955934323","https://openalex.org/W1975566260","https://openalex.org/W2069003154","https://openalex.org/W2070902649","https://openalex.org/W2075585362","https://openalex.org/W2101517004","https://openalex.org/W2104942079","https://openalex.org/W2108126284","https://openalex.org/W2110228583","https://openalex.org/W2138745909","https://openalex.org/W3101494195","https://openalex.org/W6676237016"],"related_works":["https://openalex.org/W2021850411","https://openalex.org/W4312263439","https://openalex.org/W1969481115","https://openalex.org/W3125056859","https://openalex.org/W4238075012","https://openalex.org/W1929321605","https://openalex.org/W1985630418","https://openalex.org/W1901668354","https://openalex.org/W3043663767","https://openalex.org/W2892080169"],"abstract_inverted_index":{"Online":[0],"controlled":[1,28,58,97,158],"experiments":[2,40,59,63,98,169],"are":[3,133],"often":[4,122],"utilized":[5],"to":[6,35,103,113,115,120,140,167],"make":[7],"data-driven":[8],"decisions":[9],"at":[10,19,41,60],"Amazon,":[11],"Microsoft,":[12],"eBay,":[13],"Facebook,":[14],"Google,":[15],"Yahoo,":[16],"Zynga,":[17],"and":[18,32,54,78,105,118,184,227,229],"many":[20,67],"other":[21],"companies.":[22],"While":[23],"the":[24,42,50,52,71,74,121,150,153,199,216],"theory":[25,77],"of":[26,56,62,96,108,152,157,176,201,209],"a":[27,193,206],"experiment":[29,203,225],"is":[30,80,164,182],"simple,":[31],"dates":[33],"back":[34],"Sir":[36],"Ronald":[37],"A.":[38],"Fisher's":[39],"Rothamsted":[43],"Agricultural":[44],"Experimental":[45],"Station":[46],"in":[47,49,82,85,186],"England":[48],"1920s,":[51],"deployment":[53],"mining":[55],"online":[57],"scale--thousands":[61],"now--has":[64],"taught":[65],"us":[66],"lessons.":[68],"These":[69],"exemplify":[70],"proverb":[72],"that":[73,99,170],"difference":[75],"between":[76],"practice":[79,83],"greater":[81],"than":[84],"theory.":[86],"We":[87],"present":[88],"our":[89],"learnings":[90],"as":[91],"they":[92],"happened:":[93],"puzzling":[94,131],"outcomes":[95],"we":[100,213],"analyzed":[101],"deeply":[102],"understand":[104],"explain.":[106],"Each":[107],"these":[109,130,137],"took":[110],"multiple-person":[111],"weeks":[112],"months":[114],"properly":[116],"analyze":[117],"get":[119],"surprising":[123],"root":[124,127],"cause.":[125],"The":[126,143,211],"causes":[128],"behind":[129],"results":[132,154,181,200],"not":[134,165],"isolated":[135],"incidents;":[136],"issues":[138],"generalized":[139],"multiple":[141],"experiments.":[142,159],"heightened":[144],"awareness":[145],"should":[146],"help":[147],"readers":[148],"increase":[149],"trustworthiness":[151],"coming":[155],"out":[156],"At":[160],"Microsoft's":[161],"Bing,":[162],"it":[163],"uncommon":[166],"see":[168],"impact":[171],"annual":[172],"revenue":[173],"by":[174],"millions":[175],"dollars,":[177],"thus":[178],"getting":[179],"trustworthy":[180],"critical":[183],"investing":[185],"understanding":[187],"anomalies":[188],"has":[189],"tremendous":[190],"payoff:":[191],"reversing":[192],"single":[194],"incorrect":[195],"decision":[196],"based":[197],"on":[198],"an":[202],"can":[204],"fund":[205],"whole":[207],"team":[208],"analysts.":[210],"topics":[212],"cover":[214],"include:":[215],"OEC":[217],"(Overall":[218],"Evaluation":[219],"Criterion),":[220],"click":[221],"tracking,":[222],"effect":[223],"trends,":[224],"length":[226],"power,":[228],"carryover":[230],"effects.":[231]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":48},{"year":2019,"cited_by_count":24},{"year":2018,"cited_by_count":21},{"year":2017,"cited_by_count":17},{"year":2016,"cited_by_count":20},{"year":2015,"cited_by_count":23},{"year":2014,"cited_by_count":19},{"year":2013,"cited_by_count":15},{"year":2012,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}