{"id":"https://openalex.org/W7097565573","doi":null,"title":"Trustworthy online controlled experiments: Five puzzling outcomes explained","display_name":"Trustworthy online controlled experiments: Five puzzling outcomes explained","publication_year":2012,"publication_date":"2012-01-01","ids":{"openalex":"https://openalex.org/W7097565573"},"language":"en","primary_location":{"id":"pmh:oai:CiteSeerX.psu:10.1.1.294.8275","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.294.8275","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://robotics.stanford.edu/users/ronnyk/puzzlingOutcomesInControlledExperiments.pdf","raw_type":"text"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037339239","display_name":"Ron Kohavi","orcid":"https://orcid.org/0000-0003-1623-4963","observed_orcids":["https://orcid.org/0000-0003-1623-4963"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ron Kohavi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075519075","display_name":"Alex Deng","orcid":"https://orcid.org/0000-0002-8116-5602","observed_orcids":["https://orcid.org/0000-0002-8116-5602"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alex Deng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089128609","display_name":"Brian Frasca","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Brian Frasca","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050159156","display_name":"Roger Longbotham","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Roger Longbotham","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031583581","display_name":"Toby Walker","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Toby Walker","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Ya Xu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ya Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.62942932,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.14270000159740448,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.14270000159740448,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11798","display_name":"Optimal Experimental Design Methods","score":0.08349999785423279,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.06400000303983688,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/a-b-testing","display_name":"A/B testing","score":0.9929999709129333},{"id":"https://openalex.org/keywords/root-cause-analysis","display_name":"root cause analysis","score":0.7889999747276306},{"id":"https://openalex.org/keywords/click-tracking","display_name":"click tracking","score":0.9639999866485596},{"id":"https://openalex.org/keywords/carryover-effects","display_name":"residual effects","score":0.996999979019165},{"id":"https://openalex.org/keywords/anomaly-detection","display_name":"anomaly detection","score":0.4320000112056732},{"id":"https://openalex.org/keywords/data-driven-decision-making","display_name":"data-driven decision making","score":0.5989999771118164},{"id":"https://openalex.org/keywords/microsoft-bing","display_name":"Microsoft Bing","score":0.6769999861717224}],"concepts":[{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.7788000106811523},{"id":"https://openalex.org/C171078966","wikidata":"https://www.wikidata.org/wiki/Q111029","display_name":"Root (linguistics)","level":2,"score":0.6481999754905701},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.6004999876022339},{"id":"https://openalex.org/C195487862","wikidata":"https://www.wikidata.org/wiki/Q850210","display_name":"Revenue","level":2,"score":0.5486000180244446},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.44690001010894775},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.38929998874664307},{"id":"https://openalex.org/C155108698","wikidata":"https://www.wikidata.org/wiki/Q1231081","display_name":"Randomized experiment","level":2,"score":0.367000013589859},{"id":"https://openalex.org/C2780428219","wikidata":"https://www.wikidata.org/wiki/Q16952335","display_name":"Cover (algebra)","level":2,"score":0.35100001096725464},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.33799999952316284},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.3253999948501587},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.31540000438690186},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3025999963283539},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30160000920295715},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.2685999870300293},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.26510000228881836},{"id":"https://openalex.org/C84945661","wikidata":"https://www.wikidata.org/wiki/Q7366567","display_name":"Root cause","level":2,"score":0.2637999951839447},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.2630000114440918},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.2563999891281128}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:CiteSeerX.psu:10.1.1.294.8275","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.294.8275","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://robotics.stanford.edu/users/ronnyk/puzzlingOutcomesInControlledExperiments.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2169113736","https://openalex.org/W2112508839","https://openalex.org/W2093301995","https://openalex.org/W2110228583","https://openalex.org/W2160092878","https://openalex.org/W2069003154","https://openalex.org/W3217006680","https://openalex.org/W2104942079","https://openalex.org/W3084056489","https://openalex.org/W7096070056"],"abstract_inverted_index":{"Online":[0],"controlled":[1,28,58,97,158],"experiments":[2,40,59,63,98,169],"are":[3,133],"often":[4,122],"utilized":[5],"to":[6,35,103,113,115,120,140,167],"make":[7],"datadriven":[8],"decisions":[9],"at":[10,19,41,60],"Amazon,":[11],"Microsoft,":[12],"eBay,":[13],"Facebook,":[14],"Google,":[15],"Yahoo,":[16],"Zynga,":[17],"and":[18,32,54,78,105,118,184,227,229],"many":[20,67],"other":[21],"companies.":[22],"While":[23],"the":[24,42,50,52,71,74,121,150,153,199,216],"theory":[25,77],"of":[26,56,62,96,108,152,157,176,201,209],"a":[27,193,206],"experiment":[29,203,225],"is":[30,80,164,182],"simple,":[31],"dates":[33],"back":[34],"Sir":[36],"Ronald":[37],"A.":[38],"Fisher\u2019s":[39],"Rothamsted":[43],"Agricultural":[44],"Experimental":[45],"Station":[46],"in":[47,49,82,85,186],"England":[48],"1920s,":[51],"deployment":[53],"mining":[55],"online":[57],"scale\u2014thousands":[61],"now\u2014has":[64],"taught":[65],"us":[66],"lessons.":[68],"These":[69],"exemplify":[70],"proverb":[72],"that":[73,99,170],"difference":[75],"between":[76],"practice":[79,83],"greater":[81],"than":[84],"theory.":[86],"We":[87],"present":[88],"our":[89],"learnings":[90],"as":[91],"they":[92],"happened:":[93],"puzzling":[94,131],"outcomes":[95],"we":[100,213],"analyzed":[101],"deeply":[102],"understand":[104],"explain.":[106],"Each":[107],"these":[109,130,137],"took":[110],"multiple-person":[111],"weeks":[112],"months":[114],"properly":[116],"analyze":[117],"get":[119],"surprising":[123],"root":[124,127],"cause.":[125],"The":[126,143,211],"causes":[128],"behind":[129],"results":[132,154,181,200],"not":[134,165],"isolated":[135],"incidents;":[136],"issues":[138],"generalized":[139],"multiple":[141],"experiments.":[142,159],"heightened":[144],"awareness":[145],"should":[146],"help":[147],"readers":[148],"increase":[149],"trustworthiness":[151],"coming":[155],"out":[156],"At":[160],"Microsoft\u2019s":[161],"Bing,":[162],"it":[163],"uncommon":[166],"see":[168],"impact":[171],"annual":[172],"revenue":[173],"by":[174],"millions":[175],"dollars,":[177],"thus":[178],"getting":[179],"trustworthy":[180],"critical":[183],"investing":[185],"understanding":[187],"anomalies":[188],"has":[189],"tremendous":[190],"payoff:":[191],"reversing":[192],"single":[194],"incorrect":[195],"decision":[196],"based":[197],"on":[198],"an":[202],"can":[204],"fund":[205],"whole":[207],"team":[208],"analysts.":[210],"topics":[212],"cover":[214],"include:":[215],"OEC":[217],"(Overall":[218],"Evaluation":[219],"Criterion),":[220],"click":[221],"tracking,":[222],"effect":[223],"trends,":[224],"length":[226],"power,":[228],"carryover":[230],"effects.":[231]},"counts_by_year":[],"updated_date":"2026-10-14T16:39:01.820549","created_date":"2025-10-27T00:00:00"}