{"id":"https://openalex.org/W2952127798","doi":"https://doi.org/10.1145/3292500.3330722","title":"Diagnosing Sample Ratio Mismatch in Online Controlled Experiments","display_name":"Diagnosing Sample Ratio Mismatch in Online Controlled Experiments","publication_year":2019,"publication_date":"2019-07-25","ids":{"openalex":"https://openalex.org/W2952127798","doi":"https://doi.org/10.1145/3292500.3330722","mag":"2952127798"},"language":"en","primary_location":{"id":"doi:10.1145/3292500.3330722","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3292500.3330722","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010245209","display_name":"Aleksander Fabijan","orcid":"https://orcid.org/0000-0003-4908-2708"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Aleksander Fabijan","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020286799","display_name":"Jayant Gupchup","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jayant Gupchup","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039548707","display_name":"Somit Gupta","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Somit Gupta","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010195521","display_name":"Jeff Omhover","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeff Omhover","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101625119","display_name":"Wen Qin","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wen Qin","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031956439","display_name":"Lukas Vermeer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lukas Vermeer","raw_affiliation_strings":["Booking.com, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"Booking.com, Amsterdam, Netherlands","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020697231","display_name":"Pavel Dmitriev","orcid":"https://orcid.org/0000-0001-5740-5146"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pavel Dmitriev","raw_affiliation_strings":["Outreach.io, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"Outreach.io, Seattle, WA, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5010245209"],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":5.7662,"has_fulltext":false,"cited_by_count":27,"citation_normalized_percentile":{"value":0.96249943,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"2156","last_page":"2164"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7237241864204407},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.6013057231903076},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5497007966041565},{"id":"https://openalex.org/keywords/aka","display_name":"AKA","score":0.5416013598442078},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.5328348875045776},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5039591193199158},{"id":"https://openalex.org/keywords/sample-size-determination","display_name":"Sample size determination","score":0.4494277834892273},{"id":"https://openalex.org/keywords/product","display_name":"Product (mathematics)","score":0.4475991725921631},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.41313374042510986},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3526283800601959},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.33734744787216187},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.31218916177749634},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.1656239628791809}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7237241864204407},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.6013057231903076},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5497007966041565},{"id":"https://openalex.org/C121158502","wikidata":"https://www.wikidata.org/wiki/Q4652161","display_name":"AKA","level":2,"score":0.5416013598442078},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5328348875045776},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5039591193199158},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.4494277834892273},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.4475991725921631},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.41313374042510986},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3526283800601959},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33734744787216187},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31218916177749634},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1656239628791809},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C161191863","wikidata":"https://www.wikidata.org/wiki/Q199655","display_name":"Library science","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3292500.3330722","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3292500.3330722","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.5299999713897705,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W161897729","https://openalex.org/W1557976950","https://openalex.org/W1975566260","https://openalex.org/W2031648200","https://openalex.org/W2070815407","https://openalex.org/W2080770071","https://openalex.org/W2095056536","https://openalex.org/W2110228583","https://openalex.org/W2112508839","https://openalex.org/W2162739315","https://openalex.org/W2327481811","https://openalex.org/W2405408229","https://openalex.org/W2567202186","https://openalex.org/W2584778415","https://openalex.org/W2584822570","https://openalex.org/W2726916727","https://openalex.org/W2743849295","https://openalex.org/W2751320993","https://openalex.org/W2760448772","https://openalex.org/W2761630330","https://openalex.org/W2765165237","https://openalex.org/W2884019407","https://openalex.org/W2895842676","https://openalex.org/W2896728844","https://openalex.org/W2897260156","https://openalex.org/W2897477542","https://openalex.org/W2900414111","https://openalex.org/W3104418946","https://openalex.org/W4210922477","https://openalex.org/W4300879629"],"related_works":["https://openalex.org/W4388813866","https://openalex.org/W4293088233","https://openalex.org/W2850804095","https://openalex.org/W1481656249","https://openalex.org/W2162280767","https://openalex.org/W2999104021","https://openalex.org/W2774950576","https://openalex.org/W1570799877","https://openalex.org/W2800688113","https://openalex.org/W2057598446"],"abstract_inverted_index":{"Accurately":[0],"learning":[1],"what":[2],"delivers":[3],"value":[4],"to":[5,25,40,55,138,178,184,195,279],"customers":[6],"is":[7,86,103,111,121,136,155,194],"difficult.":[8],"Online":[9],"Controlled":[10],"Experiments":[11],"(OCEs),":[12],"aka":[13],"A/B":[14],"tests,":[15],"are":[16,53],"becoming":[17],"a":[18,80,87,112,122,125,132,173,231],"standard":[19],"operating":[20],"procedure":[21],"in":[22,36,69,100,152,172,210,215,262],"software":[23,213],"companies":[24,214],"address":[26],"this":[27,192,263,276],"challenge":[28],"as":[29],"they":[30],"can":[31],"detect":[32,139],"small":[33],"causal":[34],"changes":[35],"user":[37],"behavior":[38],"due":[39],"product":[41,175],"modifications":[42],"(e.g.":[43],"new":[44],"features).":[45],"However,":[46],"like":[47,109],"any":[48],"data":[49,58,83,128],"analysis":[50],"method,":[51],"OCEs":[52,209],"sensitive":[54],"trustworthiness":[56],"and":[57,147,159,181,199,243,257,270],"quality":[59,84,129],"issues":[60,85],"which,":[61],"if":[62],"go":[63],"unaddressed":[64],"or":[65,186],"unnoticed,":[66],"may":[67,170],"result":[68,171],"making":[70,282],"wrong":[71],"decisions.":[72],"One":[73],"of":[74,79,82,117,127,191,207,223,225,236,249,273],"the":[75,93,96,101,106,144,153,163,167,255],"most":[76],"useful":[77],"indicators":[78],"variety":[81,126],"Sample":[88],"Ratio":[89],"Mismatch":[90],"(SRM)":[91],"?":[92],"situation":[94],"when":[95],"observed":[97],"sample":[98],"ratio":[99],"experiment":[102,286],"different":[104,212,218,234],"from":[105,150],"expected.":[107],"Just":[108],"fever":[110],"symptom":[113,123],"for":[114,124,233,246],"multiple":[115],"types":[116,235],"illness,":[118],"an":[119,140],"SRM":[120,164,268],"issues.":[130],"While":[131],"simple":[133],"statistical":[134],"check":[135],"used":[137,220],"SRM,":[141],"correctly":[142],"identifying":[143],"root":[145,168],"cause":[146,169],"preventing":[148,200,247],"it":[149],"happening":[151],"future":[154],"often":[156],"extremely":[157],"challenging":[158],"time":[160],"consuming.":[161],"Ignoring":[162],"without":[165],"knowing":[166],"bad":[174],"modification":[176],"appearing":[177],"be":[179],"good":[180],"getting":[182],"shipped":[183],"users,":[185],"vice":[187],"versa.":[188],"The":[189],"goal":[190],"paper":[193,264],"make":[196],"diagnosing,":[197],"fixing,":[198],"SRMs":[201,248],"easier.":[202],"Based":[203],"on":[204,284],"our":[205],"experience":[206],"running":[208],"four":[211],"over":[216],"25":[217],"products":[219],"by":[221],"hundreds":[222],"millions":[224],"users":[226],"worldwide,":[227],"we":[228,260],"have":[229],"derived":[230],"taxonomy":[232],"SRMs.":[237],"We":[238,252],"share":[239],"examples,":[240],"detection":[241],"guidelines,":[242],"best":[244],"practices":[245],"each":[250],"type.":[251],"hope":[253],"that":[254],"lessons":[256],"practical":[258],"tips":[259],"describe":[261],"will":[265],"speed":[266],"up":[267],"investigations":[269],"prevent":[271],"some":[272],"them.":[274],"Ultimately,":[275],"should":[277],"lead":[278],"improved":[280],"decision":[281],"based":[283],"trustworthy":[285],"analysis.":[287]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":2}],"updated_date":"2026-02-27T16:54:17.756197","created_date":"2025-10-10T00:00:00"}