{"id":"https://openalex.org/W4388426019","doi":"https://doi.org/10.1109/dsaa60987.2023.10302590","title":"Rapid and Scalable Bayesian AB Testing","display_name":"Rapid and Scalable Bayesian AB Testing","publication_year":2023,"publication_date":"2023-10-09","ids":{"openalex":"https://openalex.org/W4388426019","doi":"https://doi.org/10.1109/dsaa60987.2023.10302590"},"language":"en","primary_location":{"id":"doi:10.1109/dsaa60987.2023.10302590","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/dsaa60987.2023.10302590","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE 10th International Conference on Data Science and Advanced Analytics (DSAA)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041126778","display_name":"Srivas Chennu","orcid":"https://orcid.org/0000-0002-6840-2941"},"institutions":[{"id":"https://openalex.org/I4210107260","display_name":"Apple (United Kingdom)","ror":"https://ror.org/01vpeym60","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210107260"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Srivas Chennu","raw_affiliation_strings":["Apple"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple","institution_ids":["https://openalex.org/I4210107260"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080439556","display_name":"Andrew Maher","orcid":"https://orcid.org/0000-0001-5498-4509"},"institutions":[{"id":"https://openalex.org/I4210107260","display_name":"Apple (United Kingdom)","ror":"https://ror.org/01vpeym60","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210107260"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Andrew Maher","raw_affiliation_strings":["Apple"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple","institution_ids":["https://openalex.org/I4210107260"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092778216","display_name":"Christian Pangerl","orcid":null},"institutions":[{"id":"https://openalex.org/I4210107260","display_name":"Apple (United Kingdom)","ror":"https://ror.org/01vpeym60","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210107260"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Christian Pangerl","raw_affiliation_strings":["Apple"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple","institution_ids":["https://openalex.org/I4210107260"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018118001","display_name":"Subash Prabanantham","orcid":null},"institutions":[{"id":"https://openalex.org/I4210107260","display_name":"Apple (United Kingdom)","ror":"https://ror.org/01vpeym60","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210107260"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Subash Prabanantham","raw_affiliation_strings":["Apple"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple","institution_ids":["https://openalex.org/I4210107260"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034639113","display_name":"Jae Hyeon Bae","orcid":null},"institutions":[{"id":"https://openalex.org/I4210107260","display_name":"Apple (United Kingdom)","ror":"https://ror.org/01vpeym60","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210107260"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jae Hyeon Bae","raw_affiliation_strings":["Apple"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple","institution_ids":["https://openalex.org/I4210107260"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027248859","display_name":"Jamie Martin","orcid":null},"institutions":[{"id":"https://openalex.org/I4210107260","display_name":"Apple (United Kingdom)","ror":"https://ror.org/01vpeym60","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210107260"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jamie Martin","raw_affiliation_strings":["Apple"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple","institution_ids":["https://openalex.org/I4210107260"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5113040838","display_name":"Bud Goswami","orcid":null},"institutions":[{"id":"https://openalex.org/I4210107260","display_name":"Apple (United Kingdom)","ror":"https://ror.org/01vpeym60","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210107260"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Bud Goswami","raw_affiliation_strings":["Apple"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple","institution_ids":["https://openalex.org/I4210107260"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.3096,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.6381998,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":"8","issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11032","display_name":"VLSI and Analog Circuit Testing","score":0.982200026512146,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11798","display_name":"Optimal Experimental Design Methods","score":0.9804999828338623,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6945930123329163},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.5985816717147827},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.5329751968383789},{"id":"https://openalex.org/keywords/statistical-power","display_name":"Statistical power","score":0.5256941318511963},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.5194498896598816},{"id":"https://openalex.org/keywords/risk-based-testing","display_name":"Risk-based testing","score":0.5018949508666992},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.48542460799217224},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.46513503789901733},{"id":"https://openalex.org/keywords/statistical-inference","display_name":"Statistical inference","score":0.4644942581653595},{"id":"https://openalex.org/keywords/test-strategy","display_name":"Test strategy","score":0.4619126617908478},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4515870213508606},{"id":"https://openalex.org/keywords/sequential-analysis","display_name":"Sequential analysis","score":0.4350762665271759},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.40345484018325806},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3485959768295288},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.16764423251152039},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.14488539099693298},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12931859493255615},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.12690845131874084}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6945930123329163},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.5985816717147827},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5329751968383789},{"id":"https://openalex.org/C96608239","wikidata":"https://www.wikidata.org/wiki/Q1199823","display_name":"Statistical power","level":2,"score":0.5256941318511963},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.5194498896598816},{"id":"https://openalex.org/C37945671","wikidata":"https://www.wikidata.org/wiki/Q7336207","display_name":"Risk-based testing","level":5,"score":0.5018949508666992},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48542460799217224},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.46513503789901733},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.4644942581653595},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.4619126617908478},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4515870213508606},{"id":"https://openalex.org/C80478641","wikidata":"https://www.wikidata.org/wiki/Q195771","display_name":"Sequential analysis","level":2,"score":0.4350762665271759},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.40345484018325806},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3485959768295288},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.16764423251152039},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.14488539099693298},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12931859493255615},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.12690845131874084},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/dsaa60987.2023.10302590","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/dsaa60987.2023.10302590","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE 10th International Conference on Data Science and Advanced Analytics (DSAA)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.6100000143051147}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W313647156","https://openalex.org/W2105090664","https://openalex.org/W2110065044","https://openalex.org/W2126002144","https://openalex.org/W2293233856","https://openalex.org/W2466198720","https://openalex.org/W2743027853","https://openalex.org/W2744538883","https://openalex.org/W2897613819","https://openalex.org/W2943010219","https://openalex.org/W2951101733","https://openalex.org/W2963366444","https://openalex.org/W2981150028","https://openalex.org/W3047398590","https://openalex.org/W3101586444","https://openalex.org/W3175871444","https://openalex.org/W3211899127","https://openalex.org/W4225941512","https://openalex.org/W4246723808","https://openalex.org/W4255582690","https://openalex.org/W4297821805","https://openalex.org/W4388426019","https://openalex.org/W6639495887","https://openalex.org/W6755447188","https://openalex.org/W6769403462"],"related_works":["https://openalex.org/W204726053","https://openalex.org/W2915863966","https://openalex.org/W1973389480","https://openalex.org/W2287492386","https://openalex.org/W2380849986","https://openalex.org/W2362944210","https://openalex.org/W4285709722","https://openalex.org/W3037528458","https://openalex.org/W2552268348","https://openalex.org/W148386472"],"abstract_inverted_index":{"AB":[0,51,107,151,187],"testing":[1,44,74,108,121],"aids":[2],"business":[3],"operators":[4],"with":[5,63,161],"their":[6],"deci-sion":[7],"making,":[8],"and":[9,36,78,122,181],"is":[10,27],"considered":[11],"the":[12,32,37,41,55,70,79,99,143,168,193,203],"gold":[13],"standard":[14],"method":[15],"for":[16,48,75,199],"learning":[17],"from":[18,84,149],"data":[19],"to":[20,81,97,104,141,153],"improve":[21],"digital":[22],"user":[23],"experiences.":[24],"However,":[25],"there":[26],"usually":[28],"a":[29,90,162,182],"gap":[30],"between":[31,67,117],"requirements":[33],"of":[34,50,57,72,145,170,185,196],"practitioners,":[35],"constraints":[38],"imposed":[39],"by":[40,114],"statistical":[42,58,112,200],"hypothesis":[43],"methodologies":[45],"commonly":[46],"used":[47],"analysis":[49],"tests.":[52,86,156,188],"These":[53],"include":[54],"lack":[56],"power":[59,113],"in":[60,202],"multivariate":[61],"designs":[62],"many":[64],"factors,":[65,69,118],"correlations":[66,116],"these":[68,190],"need":[71],"sequential":[73,106,120],"early":[76,124],"stopping,":[77,125],"inability":[80],"pool":[82],"knowledge":[83],"past":[85,150],"Here,":[87],"we":[88,110],"propose":[89],"solution":[91],"that":[92,166],"applies":[93],"hierarchical":[94,171],"Bayesian":[95],"estimation":[96],"address":[98],"above":[100],"limitations.":[101],"In":[102],"comparison":[103],"current":[105],"methodology,":[109],"increase":[111],"exploiting":[115],"enabling":[119],"progressive":[123],"without":[126],"incurring":[127],"excessive":[128],"false":[129],"positive":[130],"risk.":[131],"We":[132,157,173],"also":[133],"demonstrate":[134,174],"how":[135],"this":[136],"methodology":[137],"can":[138],"be":[139],"extended":[140],"enable":[142],"extraction":[144],"composite":[146],"global":[147],"learnings":[148],"tests,":[152],"accelerate":[154],"future":[155],"underpin":[158],"our":[159,197],"work":[160],"solid":[163],"theoretical":[164],"framework":[165],"articulates":[167],"value":[169,195],"estimation.":[172],"its":[175],"utility":[176],"using":[177],"both":[178],"numerical":[179],"simulations":[180],"large":[183],"set":[184],"real-world":[186],"Together,":[189],"results":[191],"highlight":[192],"practical":[194],"approach":[198],"inference":[201],"technology":[204],"industry.":[205]},"counts_by_year":[{"year":2023,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}