{"id":"https://openalex.org/W2070815407","doi":"https://doi.org/10.1145/2623330.2623341","title":"Seven rules of thumb for web site experimenters","display_name":"Seven rules of thumb for web site experimenters","publication_year":2014,"publication_date":"2014-08-22","ids":{"openalex":"https://openalex.org/W2070815407","doi":"https://doi.org/10.1145/2623330.2623341","mag":"2070815407"},"language":"en","primary_location":{"id":"doi:10.1145/2623330.2623341","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2623330.2623341","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 20th ACM SIGKDD international conference on Knowledge discovery and data mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037339239","display_name":"Ron Kohavi","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ron Kohavi","raw_affiliation_strings":["Microsoft, Redmond, WA, USA","Microsoft Redmond, WA, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft Redmond, WA, USA#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075519075","display_name":"Alex Deng","orcid":"https://orcid.org/0000-0002-8116-5602"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alex Deng","raw_affiliation_strings":["Microsoft, Redmond, WA, USA","Microsoft Redmond, WA, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft Redmond, WA, USA#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050159156","display_name":"Roger Longbotham","orcid":null},"institutions":[{"id":"https://openalex.org/I4800084","display_name":"Southwest Jiaotong University","ror":"https://ror.org/00hn7w693","country_code":"CN","type":"education","lineage":["https://openalex.org/I4800084"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Roger Longbotham","raw_affiliation_strings":["SW Jiaotong University, Chengdu, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SW Jiaotong University, Chengdu, China","institution_ids":["https://openalex.org/I4800084"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5114062370","display_name":"Ya Xu","orcid":null},"institutions":[{"id":"https://openalex.org/I1316064682","display_name":"LinkedIn (United States)","ror":"https://ror.org/02fyxhe35","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I1316064682"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ya Xu","raw_affiliation_strings":["LinkedIn, Mountain View, CA, USA","[LinkedIn, Mountain View, CA, USA]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LinkedIn, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1316064682"]},{"raw_affiliation_string":"[LinkedIn, Mountain View, CA, USA]","institution_ids":["https://openalex.org/I1316064682"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":37.5776,"has_fulltext":false,"cited_by_count":232,"citation_normalized_percentile":{"value":0.99783211,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1857","last_page":"1866"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9908999800682068,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9771999716758728,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7237772345542908},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.7021288871765137},{"id":"https://openalex.org/keywords/rule-of-thumb","display_name":"Rule of thumb","score":0.7011886239051819},{"id":"https://openalex.org/keywords/web-analytics","display_name":"Web analytics","score":0.6395516395568848},{"id":"https://openalex.org/keywords/amazon-rainforest","display_name":"Amazon rainforest","score":0.6236533522605896},{"id":"https://openalex.org/keywords/web-site","display_name":"Web site","score":0.596625030040741},{"id":"https://openalex.org/keywords/revenue","display_name":"Revenue","score":0.44380050897598267},{"id":"https://openalex.org/keywords/analytics","display_name":"Analytics","score":0.42388463020324707},{"id":"https://openalex.org/keywords/web-page","display_name":"Web page","score":0.4120783805847168},{"id":"https://openalex.org/keywords/web-development","display_name":"Web development","score":0.3301785886287689},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.3120765686035156},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.30553239583969116},{"id":"https://openalex.org/keywords/web-application-security","display_name":"Web application security","score":0.17145806550979614},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.08299797773361206}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7237772345542908},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.7021288871765137},{"id":"https://openalex.org/C89246107","wikidata":"https://www.wikidata.org/wiki/Q1398821","display_name":"Rule of thumb","level":2,"score":0.7011886239051819},{"id":"https://openalex.org/C516187249","wikidata":"https://www.wikidata.org/wiki/Q10719477","display_name":"Web analytics","level":5,"score":0.6395516395568848},{"id":"https://openalex.org/C535291247","wikidata":"https://www.wikidata.org/wiki/Q177567","display_name":"Amazon rainforest","level":2,"score":0.6236533522605896},{"id":"https://openalex.org/C2984519610","wikidata":"https://www.wikidata.org/wiki/Q35127","display_name":"Web site","level":3,"score":0.596625030040741},{"id":"https://openalex.org/C195487862","wikidata":"https://www.wikidata.org/wiki/Q850210","display_name":"Revenue","level":2,"score":0.44380050897598267},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.42388463020324707},{"id":"https://openalex.org/C21959979","wikidata":"https://www.wikidata.org/wiki/Q36774","display_name":"Web page","level":2,"score":0.4120783805847168},{"id":"https://openalex.org/C79373723","wikidata":"https://www.wikidata.org/wiki/Q386275","display_name":"Web development","level":3,"score":0.3301785886287689},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.3120765686035156},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.30553239583969116},{"id":"https://openalex.org/C59241245","wikidata":"https://www.wikidata.org/wiki/Q4781497","display_name":"Web application security","level":4,"score":0.17145806550979614},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.08299797773361206},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.0},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2623330.2623341","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2623330.2623341","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 20th ACM SIGKDD international conference on Knowledge discovery and data mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W206935721","https://openalex.org/W211836331","https://openalex.org/W575598456","https://openalex.org/W583208226","https://openalex.org/W1579732355","https://openalex.org/W1969898537","https://openalex.org/W1974955023","https://openalex.org/W1975566260","https://openalex.org/W1995945562","https://openalex.org/W2038121406","https://openalex.org/W2064024870","https://openalex.org/W2069003154","https://openalex.org/W2102945991","https://openalex.org/W2104942079","https://openalex.org/W2108126284","https://openalex.org/W2110228583","https://openalex.org/W2112508839","https://openalex.org/W2138745909","https://openalex.org/W2144981148","https://openalex.org/W2160092878","https://openalex.org/W2169113736","https://openalex.org/W2752128027","https://openalex.org/W2802050749","https://openalex.org/W3101494195","https://openalex.org/W4248033998","https://openalex.org/W4298081798","https://openalex.org/W6676237016"],"related_works":["https://openalex.org/W2167278502","https://openalex.org/W2734818785","https://openalex.org/W2794551092","https://openalex.org/W2389737178","https://openalex.org/W605324233","https://openalex.org/W2982387199","https://openalex.org/W2359779378","https://openalex.org/W2068813697","https://openalex.org/W1488511360","https://openalex.org/W1980745463"],"abstract_inverted_index":{"Web":[0],"site":[1],"owners,":[2],"from":[3,30,67],"small":[4],"web":[5,24,83],"sites":[6],"to":[7,21,36],"the":[8],"largest":[9],"properties":[10],"that":[11,76],"include":[12],"Amazon,":[13,48],"Facebook,":[14],"Google,":[15],"LinkedIn,":[16,50],"Microsoft,":[17],"and":[18,51,70,85,97],"Yahoo,":[19],"attempt":[20],"improve":[22],"their":[23,71],"sites,":[25],"optimizing":[26],"for":[27,61],"criteria":[28],"ranging":[29],"repeat":[31],"usage,":[32],"time":[33],"on":[34],"site,":[35],"revenue.":[37],"Having":[38],"been":[39],"involved":[40],"in":[41,82,98],"running":[42],"thousands":[43],"of":[44,59,88],"controlled":[45,89],"experiments":[46,69],"at":[47],"Booking.com,":[49],"multiple":[52],"Microsoft":[53],"properties,":[54],"we":[55,64,77],"share":[56],"seven":[57],"rules":[58],"thumb":[60],"experimenters,":[62],"which":[63],"have":[65,79],"generalized":[66],"these":[68],"results.":[72],"These":[73],"are":[74,93,102],"principles":[75],"believe":[78],"broad":[80],"applicability":[81],"optimization":[84],"analytics":[86],"outside":[87],"experiments,":[90],"yet":[91],"they":[92],"not":[94],"provably":[95],"correct,":[96],"some":[99],"cases":[100],"exceptions":[101],"known.":[103]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":25},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":18},{"year":2021,"cited_by_count":13},{"year":2020,"cited_by_count":44},{"year":2019,"cited_by_count":31},{"year":2018,"cited_by_count":29},{"year":2017,"cited_by_count":20},{"year":2016,"cited_by_count":12},{"year":2015,"cited_by_count":18},{"year":2014,"cited_by_count":1},{"year":2013,"cited_by_count":1},{"year":2012,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}