{"id":"https://openalex.org/W2110228583","doi":"https://doi.org/10.1007/s10618-008-0114-1","title":"Controlled experiments on the web: survey and practical guide","display_name":"Controlled experiments on the web: survey and practical guide","publication_year":2008,"publication_date":"2008-07-29","ids":{"openalex":"https://openalex.org/W2110228583","doi":"https://doi.org/10.1007/s10618-008-0114-1","mag":"2110228583"},"language":"en","primary_location":{"id":"doi:10.1007/s10618-008-0114-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10618-008-0114-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10618-008-0114-1.pdf","source":{"id":"https://openalex.org/S121920818","display_name":"Data Mining and Knowledge Discovery","issn_l":"1384-5810","issn":["1384-5810","1573-756X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Data Mining and Knowledge Discovery","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10618-008-0114-1.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037339239","display_name":"Ron Kohavi","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ron Kohavi","raw_affiliation_strings":["Microsoft, One Microsoft Way, Redmond, WA, 98052, USA","Microsoft, One Microsoft Way, Redmond, USA 98052#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, One Microsoft Way, Redmond, WA, 98052, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft, One Microsoft Way, Redmond, USA 98052#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050159156","display_name":"Roger Longbotham","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Roger Longbotham","raw_affiliation_strings":["Microsoft, One Microsoft Way, Redmond, WA, 98052, USA","Microsoft, One Microsoft Way, Redmond, USA 98052#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, One Microsoft Way, Redmond, WA, 98052, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft, One Microsoft Way, Redmond, USA 98052#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091064435","display_name":"Dan Sommerfield","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dan Sommerfield","raw_affiliation_strings":["Microsoft, One Microsoft Way, Redmond, WA, 98052, USA","Microsoft, One Microsoft Way, Redmond, USA 98052#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, One Microsoft Way, Redmond, WA, 98052, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft, One Microsoft Way, Redmond, USA 98052#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5042252586","display_name":"Randal M. Henne","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Randal M. Henne","raw_affiliation_strings":["Microsoft, One Microsoft Way, Redmond, WA, 98052, USA","Microsoft, One Microsoft Way, Redmond, USA 98052#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, One Microsoft Way, Redmond, WA, 98052, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft, One Microsoft Way, Redmond, USA 98052#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5037339239"],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":{"value":2990,"currency":"USD","value_usd":2990},"apc_paid":{"value":2990,"currency":"USD","value_usd":2990},"fwci":6.9372,"has_fulltext":true,"cited_by_count":690,"citation_normalized_percentile":{"value":0.97181255,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"18","issue":"1","first_page":"140","last_page":"181"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9801999926567078,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9801999926567078,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9599999785423279,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13283","display_name":"Mental Health Research Topics","score":0.95169997215271,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7397807836532593},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.5149074792861938},{"id":"https://openalex.org/keywords/sample-size-determination","display_name":"Sample size determination","score":0.5050776600837708},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.47989022731781006},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.46952953934669495},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.41758018732070923},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3848649561405182},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3551207482814789},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3274712562561035},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.11784374713897705}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7397807836532593},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5149074792861938},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.5050776600837708},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.47989022731781006},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.46952953934669495},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.41758018732070923},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3848649561405182},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3551207482814789},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3274712562561035},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.11784374713897705},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s10618-008-0114-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10618-008-0114-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10618-008-0114-1.pdf","source":{"id":"https://openalex.org/S121920818","display_name":"Data Mining and Knowledge Discovery","issn_l":"1384-5810","issn":["1384-5810","1573-756X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Data Mining and Knowledge Discovery","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s10618-008-0114-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10618-008-0114-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10618-008-0114-1.pdf","source":{"id":"https://openalex.org/S121920818","display_name":"Data Mining and Knowledge Discovery","issn_l":"1384-5810","issn":["1384-5810","1573-756X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Data Mining and Knowledge Discovery","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2110228583.pdf","grobid_xml":"https://content.openalex.org/works/W2110228583.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W859447","https://openalex.org/W124884934","https://openalex.org/W219315687","https://openalex.org/W392079309","https://openalex.org/W1506214197","https://openalex.org/W1541055999","https://openalex.org/W1553296127","https://openalex.org/W1557935490","https://openalex.org/W1568732389","https://openalex.org/W1580090600","https://openalex.org/W1583640058","https://openalex.org/W1598186822","https://openalex.org/W1955934323","https://openalex.org/W1965844053","https://openalex.org/W1974955023","https://openalex.org/W2006901874","https://openalex.org/W2017267853","https://openalex.org/W2069003154","https://openalex.org/W2101517004","https://openalex.org/W2111322878","https://openalex.org/W2165962877","https://openalex.org/W2169879386","https://openalex.org/W2286688177","https://openalex.org/W2494251579","https://openalex.org/W2615638395","https://openalex.org/W2992025691","https://openalex.org/W4248033998","https://openalex.org/W4253736380"],"related_works":["https://openalex.org/W4388813866","https://openalex.org/W4293088233","https://openalex.org/W2850804095","https://openalex.org/W1481656249","https://openalex.org/W2162280767","https://openalex.org/W2999104021","https://openalex.org/W2774950576","https://openalex.org/W1570799877","https://openalex.org/W2800688113","https://openalex.org/W2057598446"],"abstract_inverted_index":{"The":[0],"web":[1],"provides":[2],"an":[3],"unprecedented":[4],"opportunity":[5],"to":[6,57,85,89,131,191,204],"evaluate":[7,157],"ideas":[8],"quickly":[9],"using":[10,187],"controlled":[11,101,113,217,255],"experiments,":[12,16,60,114],"also":[13],"called":[14],"randomized":[15],"A/B":[17],"tests":[18],"(and":[19],"their":[20,47,86,117,152,225],"generalizations),":[21],"split":[22],"tests,":[23,25],"Control/Treatment":[24],"MultiVariable":[26],"Tests":[27],"(MVT)":[28],"and":[29,46,76,115,121,138,150,154,159,207,230,242],"parallel":[30],"flights.":[31],"Controlled":[32,175],"experiments":[33,102,176,218],"embody":[34],"the":[35,66,90,108,196,199],"best":[36],"scientific":[37],"design":[38],"for":[39,140,147],"establishing":[40],"a":[41,54,209],"causal":[42],"relationship":[43],"between":[44],"changes":[45],"influence":[48],"on":[49,125,234],"user-observable":[50],"behavior.":[51],"We":[52,96,106,123,143,156],"provide":[53,97],"practical":[55,237],"guide":[56,65],"conducting":[58],"online":[59],"where":[61],"end-users":[62],"can":[63,184,223],"help":[64,250],"development":[67,82],"of":[68,100,111,181,195,201,212],"features.":[69],"Our":[70],"experience":[71,238],"indicates":[72],"that":[73,128,215,248],"significant":[74],"learning":[75],"return-on-investment":[77],"(ROI)":[78],"are":[79,129,165],"seen":[80],"when":[81],"teams":[83],"listen":[84],"customers,":[87],"not":[88,166],"Highest":[91],"Paid":[92],"Person\u2019s":[93],"Opinion":[94],"(HiPPO).":[95],"several":[98,126],"examples":[99],"with":[103,219,227,239],"surprising":[104],"results.":[105],"review":[107],"important":[109],"ingredients":[110],"running":[112,253],"discuss":[116],"limitations":[118],"(both":[119],"technical":[120],"organizational).":[122],"focus":[124],"areas":[127],"critical":[130],"experimentation,":[132],"including":[133],"statistical":[134],"power,":[135],"sample":[136],"size,":[137],"techniques":[139,190],"variance":[141],"reduction.":[142],"describe":[144],"common":[145],"architectures":[146],"experimentation":[148],"systems":[149,226,241],"analyze":[151],"advantages":[153],"disadvantages.":[155],"randomization":[158],"hashing":[160],"techniques,":[161],"which":[162,183],"we":[163,244],"show":[164],"as":[167,171],"simple":[168],"in":[169,252],"practice":[170],"is":[172],"often":[173],"assumed.":[174],"typically":[177],"generate":[178],"large":[179],"amounts":[180],"data,":[182],"be":[185],"analyzed":[186],"data":[188],"mining":[189],"gain":[192],"deeper":[193],"understanding":[194],"factors":[197],"influencing":[198],"outcome":[200],"interest,":[202],"leading":[203],"new":[205],"hypotheses":[206],"creating":[208],"virtuous":[210],"cycle":[211],"improvements.":[213],"Organizations":[214],"embrace":[216],"clear":[220],"evaluation":[221],"criteria":[222],"evolve":[224],"automated":[228],"optimizations":[229],"real-time":[231],"analyses.":[232],"Based":[233],"our":[235],"extensive":[236],"multiple":[240],"organizations,":[243],"share":[245],"key":[246],"lessons":[247],"will":[249],"practitioners":[251],"trustworthy":[254],"experiments.":[256]},"counts_by_year":[{"year":2026,"cited_by_count":18},{"year":2025,"cited_by_count":31},{"year":2024,"cited_by_count":41},{"year":2023,"cited_by_count":35},{"year":2022,"cited_by_count":40},{"year":2021,"cited_by_count":49},{"year":2020,"cited_by_count":64},{"year":2019,"cited_by_count":55},{"year":2018,"cited_by_count":60},{"year":2017,"cited_by_count":48},{"year":2016,"cited_by_count":55},{"year":2015,"cited_by_count":57},{"year":2014,"cited_by_count":52},{"year":2013,"cited_by_count":41},{"year":2012,"cited_by_count":20}],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2025-10-10T00:00:00"}