{"id":"https://openalex.org/W2741888629","doi":"https://doi.org/10.1145/3077136.3096468","title":"Machine Learning Powered A/B Testing","display_name":"Machine Learning Powered A/B Testing","publication_year":2017,"publication_date":"2017-07-28","ids":{"openalex":"https://openalex.org/W2741888629","doi":"https://doi.org/10.1145/3077136.3096468","mag":"2741888629"},"language":"en","primary_location":{"id":"doi:10.1145/3077136.3096468","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3077136.3096468","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069551961","display_name":"Pavel Serdyukov","orcid":null},"institutions":[{"id":"https://openalex.org/I58957048","display_name":"Yandex (Russia)","ror":"https://ror.org/04dbch786","country_code":"RU","type":"company","lineage":["https://openalex.org/I58957048"]}],"countries":["RU"],"is_corresponding":true,"raw_author_name":"Pavel Serdyukov","raw_affiliation_strings":["Yandex, Moscow, Russian Fed"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yandex, Moscow, Russian Fed","institution_ids":["https://openalex.org/I58957048"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5069551961"],"corresponding_institution_ids":["https://openalex.org/I58957048"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.10474006,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"1365","last_page":"1365"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10064","display_name":"Complex Network Analysis Techniques","score":0.9891999959945679,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10064","display_name":"Complex Network Analysis Techniques","score":0.9891999959945679,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9714000225067139,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13307","display_name":"Diverse Scientific and Engineering Research","score":0.9625999927520752,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.8798495531082153},{"id":"https://openalex.org/keywords/sensitivity","display_name":"Sensitivity (control systems)","score":0.6430336236953735},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5973560810089111},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.5779969692230225},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5666366219520569},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5534496307373047},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.5396475791931152},{"id":"https://openalex.org/keywords/performance-metric","display_name":"Performance metric","score":0.5054094791412354},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4813576340675354},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.45386576652526855},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4412652552127838},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.32141217589378357},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08890169858932495}],"concepts":[{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.8798495531082153},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.6430336236953735},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5973560810089111},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.5779969692230225},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5666366219520569},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5534496307373047},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5396475791931152},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.5054094791412354},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4813576340675354},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.45386576652526855},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4412652552127838},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.32141217589378357},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08890169858932495},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C24326235","wikidata":"https://www.wikidata.org/wiki/Q126095","display_name":"Electronic engineering","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3077136.3096468","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3077136.3096468","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":2,"referenced_works":["https://openalex.org/W2517816274","https://openalex.org/W2604305881"],"related_works":["https://openalex.org/W4361804730","https://openalex.org/W2142113611","https://openalex.org/W2334467465","https://openalex.org/W2018387840","https://openalex.org/W2087870008","https://openalex.org/W2045629210","https://openalex.org/W2162534555","https://openalex.org/W2752178021","https://openalex.org/W2107419853","https://openalex.org/W2143024819"],"abstract_inverted_index":{"Online":[0],"search":[1,14,32],"evaluation,":[2],"and":[3,25,64,152,197,261,288,307,343,364,399,496],"A/B":[4,83,201,243],"testing":[5],"in":[6,82,117,159,187,257,268,273],"particular,":[7],"is":[8,47,120,464,479],"an":[9,242],"irreplaceable":[10],"tool":[11],"for":[12,20,38,78,93,284,324,336,348,474],"modern":[13],"engines.":[15],"Typically,":[16],"online":[17],"experiments":[18,119,162,202],"last":[19],"several":[21,248],"days":[22,240],"or":[23,206],"weeks":[24],"require":[26],"a":[27,70,95,99,106,123,134,149,165,172,212,219,224,290,325,401,446,498],"considerable":[28],"portion":[29],"of":[30,44,55,72,104,125,137,185,194,209,223,234,241,254,281,320,329,353,379,387,425,431,441,445,461,469,482],"the":[31,35,42,53,91,142,156,160,176,183,188,195,230,235,239,252,255,266,269,279,299,305,321,333,340,345,354,361,369,377,385,388,396,412,423,426,432,442,459,467,471,475,483,504],"traffic.":[33],"Despite":[34],"increasing":[36],"need":[37,77],"running":[39],"more":[40],"experiments,":[41],"amount":[43],"that":[45,113,141,218,250,293,428,478],"traffic":[46,214],"limited.":[48],"This":[49],"situation":[50],"leads":[51],"to":[52,74,164,199,264,296,303,368,418,438,466],"problem":[54,103,184,280,460,468],"finding":[56,94,105,470],"new":[57,402],"key":[58,322,397,447,476],"performance":[59],"metrics":[60,126,196,249,287],"with":[61,122,155,203,404],"higher":[62,173],"sensitivity":[63,174,193],"lower":[65],"variance.":[66,406],"Recently,":[67],"we":[68,132,246,277,383,391,409,416,436,490],"proposed":[69,247],"number":[71],"techniques":[73],"alleviate":[75],"this":[76,337,349,457],"larger":[79,207],"sample":[80],"sizes":[81],"experiments.":[84],"One":[85],"approach":[86,181],"was":[87],"based":[88,311,421],"on":[89,312,360,422],"formulating":[90],"quest":[92],"sensitive":[96,107],"metric":[97,108,150,225,323,398,403,477],"as":[98,148],"data-driven":[100],"machine":[101,314],"learning":[102,315],"combination":[109,136,144],"\\cite{Kharitonov2017}.":[110],"We":[111,216],"assumed":[112],"each":[114],"single":[115],"observation":[116],"these":[118,138],"assigned":[121],"vector":[124],"(features)":[127],"describing":[128],"it.":[129],"After":[130],"that,":[131],"learned":[133,143],"linear":[135],"metrics,":[139],"such":[140,258],"can":[145],"be":[146,227],"considered":[147],"itself,":[151],"(a)":[153],"agrees":[154],"preference":[157],"direction":[158],"seed":[161],"according":[163],"baseline":[166,177],"ground":[167],"truth":[168],"metric,":[169,448],"(b)":[170],"achieves":[171],"than":[175,503],"ground-truth":[178],"metric.":[179],"Another":[180],"addressed":[182,278],"delays":[186],"treatment":[189,221,341,346,362,372,433,484],"effects":[190],"causing":[191],"low":[192],"requiring":[198],"conduct":[200],"longer":[204],"duration":[205],"set":[208],"users":[210],"from":[211,395,452],"limited":[213],"\\cite{Drutsa2017}.":[215],"found":[217],"delayed":[220],"effect":[222,347],"could":[226],"revealed":[228],"through":[229],"daily":[231],"time":[232,259],"series":[233,260],"metric's":[236],"measurements":[237],"over":[238],"test.":[244],"So,":[245],"learn":[251],"models":[253],"trend":[256],"use":[262],"them":[263],"quantify":[265],"changes":[267],"user":[270,285,327,338,427],"behavior.":[271],"Finally,":[272],"another":[274],"study":[275],"\\cite{Poyarkov2016},":[276],"variance":[282,306,378,462,501],"reduction":[283,463,502],"engagement":[286],"developed":[289],"general":[291,488],"framework":[292],"allows":[294],"us":[295],"incorporate":[297],"both":[298],"existing":[300],"state-of-the-art":[301],"approaches":[302],"reduce":[304],"some":[308],"novel":[309],"ones":[310],"advanced":[313],"techniques.":[316],"The":[317,351],"expected":[318,334],"value":[319,335,386,444],"given":[326],"consists":[328],"two":[330],"components:":[331],"(1)":[332],"irrespectively":[339],"assignment":[342,363],"(2)":[344],"user.":[350],"expectation":[352],"1st":[355],"component":[356,414],"does":[357,365],"not":[358,366,480],"depend":[359],"contribute":[367],"actual":[370],"average":[371,443,450],"effect,":[373],"but":[374],"may":[375],"increase":[376],"its":[380,449,453],"estimation.":[381],"If":[382],"knew":[384],"first":[389,413],"component,":[390],"would":[392],"subtract":[393],"it":[394,420],"obtain":[400],"decreased":[405],"However,":[407],"since":[408],"cannot":[410],"evaluate":[411],"exactly,":[415],"propose":[417,437],"predict":[419],"attributes":[424],"are":[429],"independent":[430],"exposure.":[434,485],"Therefore,":[435],"utilize,":[439],"instead":[440],"deviation":[451],"predicted":[454],"value.":[455],"In":[456,486],"way,":[458],"reduced":[465],"best":[472],"predictor":[473],"aware":[481],"our":[487],"approach,":[489],"apply":[491],"gradient":[492],"boosted":[493],"decision":[494],"trees":[495],"achieve":[497],"significantly":[499],"greater":[500],"state-of-the-art.":[505]},"counts_by_year":[{"year":2022,"cited_by_count":2}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}