{"id":"https://openalex.org/W2945720420","doi":"https://doi.org/10.1145/3308560.3320093","title":"A/B Testing at Scale: Accelerating Software Innovation","display_name":"A/B Testing at Scale: Accelerating Software Innovation","publication_year":2019,"publication_date":"2019-05-13","ids":{"openalex":"https://openalex.org/W2945720420","doi":"https://doi.org/10.1145/3308560.3320093","mag":"2945720420"},"language":"en","primary_location":{"id":"doi:10.1145/3308560.3320093","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308560.3320093","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of The 2019 World Wide Web Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3308560.3320093","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039548707","display_name":"Somit Gupta","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Somit Gupta","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062940753","display_name":"Ronny Kohavi","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ronny Kohavi","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075519075","display_name":"Alex Deng","orcid":"https://orcid.org/0000-0002-8116-5602"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Alex Deng","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010195521","display_name":"Jeff Omhover","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jeff Omhover","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069503088","display_name":"Pawel Janowski","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Pawel Janowski","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3175,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.81218487,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"1299","last_page":"1300"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9890999794006348,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9509999752044678,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5946172475814819},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5857076048851013},{"id":"https://openalex.org/keywords/software-testing","display_name":"Software testing","score":0.5154963135719299},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.439532607793808},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.43313831090927124},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1907057762145996}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5946172475814819},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5857076048851013},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.5154963135719299},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.439532607793808},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.43313831090927124},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1907057762145996},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3308560.3320093","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308560.3320093","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of The 2019 World Wide Web Conference","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3308560.3320093","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308560.3320093","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of The 2019 World Wide Web Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.6399999856948853,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W1896407793","https://openalex.org/W2030660637","https://openalex.org/W2112508839","https://openalex.org/W2126002144","https://openalex.org/W2169113736","https://openalex.org/W2277232523","https://openalex.org/W2468478068","https://openalex.org/W2509860763","https://openalex.org/W2532610538","https://openalex.org/W2536489703","https://openalex.org/W2567202186","https://openalex.org/W2584778415","https://openalex.org/W2620002388","https://openalex.org/W2740607000"],"related_works":["https://openalex.org/W2090301720","https://openalex.org/W2008194781","https://openalex.org/W3089971865","https://openalex.org/W2038209453","https://openalex.org/W3176274227","https://openalex.org/W2913502950","https://openalex.org/W1916954724","https://openalex.org/W2914660298","https://openalex.org/W2122084118","https://openalex.org/W3010281314"],"abstract_inverted_index":{"The":[0,108],"Internet":[1],"and":[2,8,21,37,44,122,143,149,170,175,198],"the":[3,118,120,159],"general":[4],"digitalization":[5],"of":[6,30,106,110,124,131,138,194],"products":[7],"operations":[9],"provides":[10],"an":[11],"unprecedented":[12],"opportunity":[13],"to":[14,58,68,74,82,89,98,192],"accelerate":[15],"innovation":[16],"while":[17],"applying":[18],"a":[19,103,111,136],"rigorous":[20],"trustworthy":[22],"methodology":[23],"for":[24,117,202],"supporting":[25],"key":[26,184],"product":[27],"decisions.":[28],"Developers":[29],"connected":[31],"software,":[32],"including":[33],"web":[34,139],"sites,":[35,140],"applications,":[36],"devices,":[38],"can":[39],"now":[40,96],"evaluate":[41],"ideas":[42],"quickly":[43],"accurately":[45],"using":[46,167],"controlled":[47,93,112,126],"experiments,":[48],"also":[49],"known":[50],"as":[51],"A/B":[52,161],"tests.":[53],"From":[54],"front-end":[55],"user-interface":[56],"changes":[57],"backend":[59],"algorithms,":[60],"from":[61,187],"search":[62],"engines":[63],"(e.g.,":[64,70,78,85],"Google,":[65],"Bing,":[66],"Yahoo!)":[67],"retailers":[69],"Amazon,":[71],"eBay,":[72],"Etsy)":[73],"social":[75],"networking":[76],"services":[77,84],"Facebook,":[79],"LinkedIn,":[80],"Twitter)":[81],"travel":[83],"Expedia,":[86],"Airbnb,":[87],"Booking.com)":[88],"many":[90,147],"startups,":[91],"online":[92,125],"experiments":[94,127,195],"are":[95],"utilized":[97],"make":[99],"data-driven":[100],"decisions":[101],"at":[102,128,190],"wide":[104],"range":[105],"companies.":[107],"theory":[109],"experiment":[113],"is":[114],"simple,":[115],"but":[116],"practitioner":[119],"deployment":[121],"evaluation":[123],"scale":[129],"(100\u2019s":[130],"concurrently":[132],"running":[133],"experiments)":[134],"across":[135],"variety":[137],"mobile":[141],"apps,":[142],"desktop":[144],"applications":[145],"presents":[146],"pitfalls":[148],"new":[150],"research":[151,176],"challenges.":[152],"In":[153],"this":[154],"tutorial,":[155],"we":[156],"will":[157,182],"introduce":[158],"overall":[160],"testing":[162],"methodology,":[163],"walkthrough":[164],"use":[165],"cases":[166],"real":[168],"examples,":[169],"then":[171],"focus":[172],"on":[173],"practical":[174],"challenges":[177],"in":[178],"scaling":[179,188],"experimentation.":[180],"We":[181],"share":[183],"lessons":[185],"learned":[186],"experimentation":[189],"Microsoft":[191],"thousands":[193],"per":[196],"year":[197],"outline":[199],"promising":[200],"directions":[201],"future":[203],"work.":[204]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}