{"id":"https://openalex.org/W2069003154","doi":"https://doi.org/10.1145/1281192.1281295","title":"Practical guide to controlled experiments on the web","display_name":"Practical guide to controlled experiments on the web","publication_year":2007,"publication_date":"2007-08-12","ids":{"openalex":"https://openalex.org/W2069003154","doi":"https://doi.org/10.1145/1281192.1281295","mag":"2069003154"},"language":"en","primary_location":{"id":"doi:10.1145/1281192.1281295","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1281192.1281295","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 13th ACM SIGKDD international conference on Knowledge discovery and data mining","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037339239","display_name":"Ron Kohavi","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ron Kohavi","raw_affiliation_strings":["Microsoft, Redmond, WA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042252586","display_name":"Randal M. Henne","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Randal M. Henne","raw_affiliation_strings":["Microsoft, Redmond, WA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091064435","display_name":"Dan Sommerfield","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dan Sommerfield","raw_affiliation_strings":["Microsoft, Redmond, WA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":32.2386,"has_fulltext":false,"cited_by_count":383,"citation_normalized_percentile":{"value":0.998679,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"959","last_page":"967"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9828000068664551,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9828000068664551,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9508000016212463,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9478999972343445,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7592487335205078},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.5654008984565735},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.4971170723438263},{"id":"https://openalex.org/keywords/sample-size-determination","display_name":"Sample size determination","score":0.47027307748794556},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.44588035345077515},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.42420047521591187},{"id":"https://openalex.org/keywords/web-application","display_name":"Web application","score":0.4187147915363312},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.371135950088501},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3585028648376465},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3315467834472656},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.1448632776737213}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7592487335205078},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5654008984565735},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.4971170723438263},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.47027307748794556},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.44588035345077515},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.42420047521591187},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.4187147915363312},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.371135950088501},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3585028648376465},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3315467834472656},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.1448632776737213},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/1281192.1281295","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1281192.1281295","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 13th ACM SIGKDD international conference on Knowledge discovery and data mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W859447","https://openalex.org/W219315687","https://openalex.org/W392079309","https://openalex.org/W1512389042","https://openalex.org/W1541055999","https://openalex.org/W1553296127","https://openalex.org/W1557935490","https://openalex.org/W1580090600","https://openalex.org/W1598186822","https://openalex.org/W1955934323","https://openalex.org/W2017267853","https://openalex.org/W2101517004","https://openalex.org/W2111322878","https://openalex.org/W2169879386","https://openalex.org/W2286688177","https://openalex.org/W2615638395","https://openalex.org/W2992025691","https://openalex.org/W4253736380"],"related_works":["https://openalex.org/W4293088233","https://openalex.org/W1481656249","https://openalex.org/W2162280767","https://openalex.org/W2999104021","https://openalex.org/W2774950576","https://openalex.org/W1570799877","https://openalex.org/W2800688113","https://openalex.org/W2057598446","https://openalex.org/W4236933217","https://openalex.org/W4210818743"],"abstract_inverted_index":{"The":[0],"web":[1],"provides":[2],"an":[3],"unprecedented":[4],"opportunity":[5],"to":[6,59,87,91,133,193,206],"evaluate":[7,159],"ideas":[8],"quickly":[9],"using":[10,189],"controlled":[11,103,115,219,257],"experiments,":[12,62,116],"also":[13],"called":[14],"randomized":[15],"experiments":[16,35,104,178,220],"(single":[17],"factor":[18],"or":[19],"factorial":[20],"designs),":[21],"A/B":[22],"tests":[23],"(and":[24],"their":[25,49,88,119,154,227],"generalizations),":[26],"split":[27],"tests,":[28,30],"Control/Treatment":[29],"and":[31,48,78,117,123,140,152,156,161,209,232,244],"parallel":[32],"flights.":[33],"Controlled":[34,177],"embody":[36],"the":[37,68,92,110,198,201],"best":[38],"scientific":[39],"design":[40],"for":[41,142,149],"establishing":[42],"a":[43,56,211],"causal":[44],"relationship":[45],"between":[46],"changes":[47],"influence":[50],"on":[51,127,236],"user-observable":[52],"behavior.":[53],"We":[54,98,108,125,145,158],"provide":[55,99],"practical":[57,239],"guide":[58,67],"conducting":[60],"online":[61],"where":[63],"end-users":[64],"can":[65,186,225],"help":[66,252],"development":[69,84],"of":[70,102,113,183,197,203,214],"features.":[71],"Our":[72],"experience":[73,240],"indicates":[74],"that":[75,130,217,250],"significant":[76],"learning":[77],"return-on-investment":[79],"(ROI)":[80],"are":[81,131,167],"seen":[82],"when":[83],"teams":[85],"listen":[86],"customers,":[89],"not":[90,168],"Highest":[93],"Paid":[94],"Person's":[95],"Opinion":[96],"(HiPPO).":[97],"several":[100,128],"examples":[101],"with":[105,221,229,241],"surprising":[106],"results.":[107],"review":[109],"important":[111],"ingredients":[112],"running":[114,255],"discuss":[118],"limitations":[120],"(both":[121],"technical":[122],"organizational).":[124],"focus":[126],"areas":[129],"critical":[132],"experimentation,":[134],"including":[135],"statistical":[136],"power,":[137],"sample":[138],"size,":[139],"techniques":[141,192],"variance":[143],"reduction.":[144],"describe":[146],"common":[147],"architectures":[148],"experimentation":[150],"systems":[151,228,243],"analyze":[153],"advantages":[155],"disadvantages.":[157],"randomization":[160],"hashing":[162],"techniques,":[163],"which":[164,185],"we":[165,246],"show":[166],"as":[169,173],"simple":[170],"in":[171,254],"practice":[172],"is":[174],"often":[175],"assumed.":[176],"typically":[179],"generate":[180],"large":[181],"amounts":[182],"data,":[184],"be":[187],"analyzed":[188],"data":[190],"mining":[191],"gain":[194],"deeper":[195],"understanding":[196],"factors":[199],"influencing":[200],"outcome":[202],"interest,":[204],"leading":[205],"new":[207],"hypotheses":[208],"creating":[210],"virtuous":[212],"cycle":[213],"improvements.":[215],"Organizations":[216],"embrace":[218],"clear":[222],"evaluation":[223],"criteria":[224],"evolve":[226],"automated":[230],"optimizations":[231],"real-time":[233],"analyses.":[234],"Based":[235],"our":[237],"extensive":[238],"multiple":[242],"organizations,":[245],"share":[247],"key":[248],"lessons":[249],"will":[251],"practitioners":[253],"trustworthy":[256],"experiments.":[258]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":57},{"year":2024,"cited_by_count":13},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":18},{"year":2020,"cited_by_count":12},{"year":2019,"cited_by_count":21},{"year":2018,"cited_by_count":20},{"year":2017,"cited_by_count":25},{"year":2016,"cited_by_count":28},{"year":2015,"cited_by_count":24},{"year":2014,"cited_by_count":27},{"year":2013,"cited_by_count":32},{"year":2012,"cited_by_count":20}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}