{"id":"https://openalex.org/W2810165363","doi":"https://doi.org/10.1145/3219166.3219204","title":"The A/B Testing Problem","display_name":"The A/B Testing Problem","publication_year":2018,"publication_date":"2018-06-11","ids":{"openalex":"https://openalex.org/W2810165363","doi":"https://doi.org/10.1145/3219166.3219204","mag":"2810165363"},"language":"en","primary_location":{"id":"doi:10.1145/3219166.3219204","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3219166.3219204","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 ACM Conference on Economics and Computation","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063791392","display_name":"Eduardo M. Azevedo","orcid":"https://orcid.org/0000-0001-6048-6506"},"institutions":[{"id":"https://openalex.org/I4210157222","display_name":"William P. Wharton Trust","ror":"https://ror.org/05vea6821","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210157222"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Eduardo M. Azevedo","raw_affiliation_strings":["Wharton, Philadelphia, PA, USA"],"affiliations":[{"raw_affiliation_string":"Wharton, Philadelphia, PA, USA","institution_ids":["https://openalex.org/I4210157222"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075519075","display_name":"Alex Deng","orcid":"https://orcid.org/0000-0002-8116-5602"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alex Deng","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090612972","display_name":"Jos\u00e9 Luis Montiel Olea","orcid":"https://orcid.org/0000-0002-1037-1628"},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jose Luis Montiel Olea","raw_affiliation_strings":["Columbia University, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Columbia University, New York, NY, USA","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079845832","display_name":"Justin M. Rao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Justin Rao","raw_affiliation_strings":["Homeaway, Austin, TX, USA"],"affiliations":[{"raw_affiliation_string":"Homeaway, Austin, TX, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5086938626","display_name":"E. Glen Weyl","orcid":"https://orcid.org/0000-0001-8824-6977"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"E. Glen Weyl","raw_affiliation_strings":["Microsoft Research, Cambridge, MA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Cambridge, MA, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5063791392"],"corresponding_institution_ids":["https://openalex.org/I4210157222"],"apc_list":null,"apc_paid":null,"fwci":0.6515,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.76346331,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"461","last_page":"462"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11798","display_name":"Optimal Experimental Design Methods","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.991100013256073,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6274283528327942},{"id":"https://openalex.org/keywords/product","display_name":"Product (mathematics)","score":0.5146985650062561},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.49289822578430176},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.49052804708480835},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.4868559241294861},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.47660893201828003},{"id":"https://openalex.org/keywords/outlier","display_name":"Outlier","score":0.465806782245636},{"id":"https://openalex.org/keywords/new-product-development","display_name":"New product development","score":0.44653642177581787},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3635595142841339},{"id":"https://openalex.org/keywords/marketing","display_name":"Marketing","score":0.3317885994911194},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.2182927429676056},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14879220724105835},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1415574848651886},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.1307099461555481}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6274283528327942},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.5146985650062561},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.49289822578430176},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.49052804708480835},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.4868559241294861},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.47660893201828003},{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.465806782245636},{"id":"https://openalex.org/C19351080","wikidata":"https://www.wikidata.org/wiki/Q1395034","display_name":"New product development","level":2,"score":0.44653642177581787},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3635595142841339},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.3317885994911194},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.2182927429676056},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14879220724105835},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1415574848651886},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.1307099461555481},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3219166.3219204","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3219166.3219204","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 ACM Conference on Economics and Computation","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.6399999856948853,"id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":3,"referenced_works":["https://openalex.org/W2039522160","https://openalex.org/W2329632867","https://openalex.org/W4237910115"],"related_works":["https://openalex.org/W3006513224","https://openalex.org/W2046456988","https://openalex.org/W2357409937","https://openalex.org/W2510582230","https://openalex.org/W2978674666","https://openalex.org/W2074430941","https://openalex.org/W2113096305","https://openalex.org/W1977636359","https://openalex.org/W2772305933","https://openalex.org/W2580722822"],"abstract_inverted_index":{"\"Randomized":[0],"experiments":[1,19,99,340],"are":[2,68,387],"increasingly":[3],"central":[4],"to":[5,39,59,100,139,203,207,269,299,301,315,329,339,368,374],"innovation":[6,132,233,275,306],"in":[7,76],"many":[8,115,126,145],"fields.":[9],"In":[10,44,113,163],"the":[11,37,45,189,209,220,227,230,316,330,333,345,355,358,360,376,380,384],"tech":[12,81],"sector,":[13],"major":[14],"platforms":[15],"run":[16],"thousands":[17],"of":[18,27,29,98,109,170,200,211,229,232,243,265,274,296,319,336,348,357,366,379,383],"(called":[20],"A/B":[21,190,254,303],"tests)":[22],"each":[23,169],"year":[24],"on":[25,85,226],"tens":[26],"millions":[28],"users":[30,297,337,349],"at":[31],"any":[32,131],"given":[33],"time":[34],"and":[35,47,53,63,80,117,129,141,156,205,213,235,279,322],"use":[36,55],"results":[38],"screen":[40],"most":[41],"product":[42],"innovations.":[43,290],"policy":[46,111],"academic":[48],"circles,":[49],"governments,":[50],"nonprofit":[51],"organizations,":[52],"academics":[54],"randomized":[56],"control":[57],"trials":[58],"evaluate":[60],"social":[61],"programs":[62],"shape":[64],"public":[65],"policy.":[66],"Experiments":[67],"not":[69,154],"only":[70],"prevalent,":[71],"but":[72],"also":[73,195],"highly":[74],"heterogeneous":[75],"design.":[77],"Policy":[78],"makers":[79],"giants":[82],"typically":[83],"focus":[84],"a":[86,95,110,120,180,246,263,282,294,310,363],"\"go":[87,122],"big\"":[88],"approach,":[89,124],"obtaining":[90],"large":[91,248],"sample":[92],"sizes":[93],"for":[94,183,351],"small":[96,107,127],"number":[97,295,335,346],"ensure":[101],"they":[102],"that":[103,152,160,186,219,332,386],"can":[104],"detect":[105],"even":[106],"benefits":[108],"intervention.":[112],"contrast,":[114],"start-ups":[116],"entrepreneurs":[118],"take":[119],"different":[121],"lean\"":[123],"running":[125],"tests":[128],"discarding":[130],"without":[133],"outstanding":[134],"success.":[135],"The":[136,193,215,253,271,291,325,370],"idea":[137,320],"is":[138,173,218,257,277,286,327,341,373],"quickly":[140],"cheaply":[142],"experiment":[143],"with":[144,245,312],"ideas,":[146],"abandon":[147],"or":[148,250],"pivot":[149],"from":[150,281],"ideas":[151,159,367,385],"do":[153,161,176],"work,":[155],"scale":[157],"up":[158],"work.":[162],"this":[164],"paper,":[165],"we":[166,178,187],"study":[167],"when":[168],"these":[171,237],"approaches":[172],"appropriate.":[174],"To":[175],"so,":[177],"propose":[179],"new":[181],"framework":[182],"optimal":[184,198,221],"experimentation":[185,222],"call":[188],"testing":[191,255],"problem.":[192],"frameworks":[194],"yields":[196,309],"an":[197,302],"strategy":[199,223],"what":[201],"innovations":[202,244,267],"implement":[204],"methods":[206],"calculate":[208],"value":[210],"data":[212],"experimentation.":[214,352],"key":[216],"insight":[217],"depends":[224],"crucially":[225],"tails":[228,238],"distribution":[231,283],"quality,":[234],"whether":[236],"have":[239],"\"black":[240],"swan\"":[241],"outliers,":[242],"very":[247],"positive":[249],"negative":[251],"impact.":[252],"problem":[256],"as":[258],"follows.":[259],"A":[260],"firm":[261,292,326,361],"has":[262],"set":[264],"potential":[266],"i=1,-,I":[268],"implement.":[270,369],"quality":[272,318,382],"..._i":[273],"i":[276,321],"unknown":[278],"comes":[280],"G.":[284],"Quality":[285],"independently":[287],"distributed":[288],"across":[289],"selects":[293,362],"n_i":[298],"allocate":[300],"test":[304],"evaluating":[305],"i.":[307],"This":[308],"signal":[311],"mean":[313],"equal":[314],"true":[317,381],"variance":[323],"a^2/n_i.":[324],"subject":[328],"constraint":[331],"total":[334],"assigned":[338],"no":[342],"greater":[343],"than":[344],"N":[347],"available":[350],"After":[353],"seeing":[354],"realization":[356],"signals,":[359],"subset":[364],"S":[365],"firm's":[371],"objective":[372],"maximize":[375],"expected":[377],"sum":[378],"implemented.":[388]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}