{"id":"https://openalex.org/W2950707656","doi":"https://doi.org/10.1145/3314183.3323853","title":"Experimentation Pitfalls to Avoid in A/B Testing for Online Personalization","display_name":"Experimentation Pitfalls to Avoid in A/B Testing for Online Personalization","publication_year":2019,"publication_date":"2019-06-06","ids":{"openalex":"https://openalex.org/W2950707656","doi":"https://doi.org/10.1145/3314183.3323853","mag":"2950707656"},"language":"en","primary_location":{"id":"doi:10.1145/3314183.3323853","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3314183.3323853","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Adjunct Publication of the 27th Conference on User Modeling, Adaptation and Personalization","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027112360","display_name":"Mar\u00eda Esteller-Cucala","orcid":"https://orcid.org/0000-0001-7982-6822"},"institutions":[{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"Maria Esteller-Cucala","raw_affiliation_strings":["Universitat Politecnica de Catalunya-Barcelona Technology & SEAT, S.A., Barcelona, Spain"],"affiliations":[{"raw_affiliation_string":"Universitat Politecnica de Catalunya-Barcelona Technology & SEAT, S.A., Barcelona, Spain","institution_ids":["https://openalex.org/I9617848"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068269176","display_name":"Vicen\u00e7 Fern\u00e1ndez","orcid":"https://orcid.org/0000-0001-5187-5024"},"institutions":[{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Vicenc Fernandez","raw_affiliation_strings":["Universitat Polit\u00e8cnica de Catalunya-Barcelona Technology, Barcelona, Spain"],"affiliations":[{"raw_affiliation_string":"Universitat Polit\u00e8cnica de Catalunya-Barcelona Technology, Barcelona, Spain","institution_ids":["https://openalex.org/I9617848"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5012328526","display_name":"Diego Villuendas","orcid":"https://orcid.org/0000-0002-5484-3199"},"institutions":[{"id":"https://openalex.org/I1293796582","display_name":"Seat (Spain)","ror":"https://ror.org/024kx0v68","country_code":"ES","type":"company","lineage":["https://openalex.org/I1293796582"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Diego Villuendas","raw_affiliation_strings":["SEAT, S.A., Barcelona, Spain"],"affiliations":[{"raw_affiliation_string":"SEAT, S.A., Barcelona, Spain","institution_ids":["https://openalex.org/I1293796582"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5027112360"],"corresponding_institution_ids":["https://openalex.org/I9617848"],"apc_list":null,"apc_paid":null,"fwci":0.8682,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.86654981,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"153","last_page":"159"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10068","display_name":"Technology Adoption and User Behaviour","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10068","display_name":"Technology Adoption and User Behaviour","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12238","display_name":"Green IT and Sustainability","score":0.982699990272522,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9824000000953674,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/personalization","display_name":"Personalization","score":0.8925919532775879},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8030092716217041},{"id":"https://openalex.org/keywords/argumentation-theory","display_name":"Argumentation theory","score":0.6110669374465942},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5890659689903259},{"id":"https://openalex.org/keywords/economic-shortage","display_name":"Economic shortage","score":0.5794401168823242},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5728858709335327},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.44198864698410034},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.4200797975063324},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.20690765976905823},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.17346355319023132}],"concepts":[{"id":"https://openalex.org/C183003079","wikidata":"https://www.wikidata.org/wiki/Q1000371","display_name":"Personalization","level":2,"score":0.8925919532775879},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8030092716217041},{"id":"https://openalex.org/C65059942","wikidata":"https://www.wikidata.org/wiki/Q270105","display_name":"Argumentation theory","level":2,"score":0.6110669374465942},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5890659689903259},{"id":"https://openalex.org/C194051981","wikidata":"https://www.wikidata.org/wiki/Q1337691","display_name":"Economic shortage","level":3,"score":0.5794401168823242},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5728858709335327},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.44198864698410034},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.4200797975063324},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.20690765976905823},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.17346355319023132},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C2778137410","wikidata":"https://www.wikidata.org/wiki/Q2732820","display_name":"Government (linguistics)","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3314183.3323853","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3314183.3323853","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Adjunct Publication of the 27th Conference on User Modeling, Adaptation and Personalization","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W1700132089","https://openalex.org/W2005923298","https://openalex.org/W2010057062","https://openalex.org/W2043002652","https://openalex.org/W2069003154","https://openalex.org/W2070815407","https://openalex.org/W2074380434","https://openalex.org/W2081556727","https://openalex.org/W2095056536","https://openalex.org/W2104942079","https://openalex.org/W2105783544","https://openalex.org/W2123039127","https://openalex.org/W2169113736","https://openalex.org/W2311766271","https://openalex.org/W2584778415","https://openalex.org/W2588594887","https://openalex.org/W2608333376","https://openalex.org/W2743849295","https://openalex.org/W2744538883","https://openalex.org/W2897260156","https://openalex.org/W2963366444","https://openalex.org/W2963805801","https://openalex.org/W3124380021"],"related_works":["https://openalex.org/W4256185029","https://openalex.org/W2141332034","https://openalex.org/W2337488240","https://openalex.org/W2762454042","https://openalex.org/W4388437769","https://openalex.org/W2064153856","https://openalex.org/W52290235","https://openalex.org/W3007362904","https://openalex.org/W1598849238","https://openalex.org/W756293569"],"abstract_inverted_index":{"Online":[0],"controlled":[1,85,157],"experiments":[2,36,86],"(also":[3],"called":[4],"A/B":[5],"tests,":[6],"bucket":[7],"testing":[8,104],"or":[9,69],"randomized":[10],"experiments)":[11],"have":[12,51],"become":[13],"an":[14],"habitual":[15],"practice":[16],"in":[17,75,149,165],"numerous":[18],"companies":[19],"for":[20,121],"measuring":[21],"the":[22,40,45,65,76,94,100,114,176,187],"impact":[23],"of":[24,39,67,80,103,113,133,155,179,186],"new":[25,49],"features":[26,50,152],"and":[27,90,184],"changes":[28],"deployed":[29],"to":[30,43,163,174,181],"softwares":[31],"products.":[32],"In":[33,55,107],"theory,":[34],"these":[35,88],"are":[37,59,118,160],"one":[38],"simplest":[41],"methods":[42],"evaluate":[44],"potential":[46],"effects":[47],"that":[48,62,117],"on":[52,83],"user's":[53],"behavior.":[54],"practice,":[56],"however,":[57],"there":[58,145,159],"many":[60],"pitfalls":[61,89,116,162],"can":[63],"obscure":[64],"interpretation":[66],"results":[68],"induce":[70],"invalid":[71],"conclusions.":[72],"There":[73],"is,":[74],"literature,":[77],"no":[78],"shortage":[79],"prior":[81],"work":[82],"online":[84,156],"addressing":[87],"conclusions":[91],"misinterpretations,":[92],"but":[93],"topic":[95],"is":[96,146],"not":[97],"tackled":[98],"considering":[99],"specific":[101],"case":[102],"personalization":[105,122],"features.":[106,123],"this":[108,170],"paper,":[109,171],"we":[110,129,172],"present":[111],"some":[112,161],"experimentation":[115],"particularly":[119],"important":[120],"To":[124],"better":[125],"illustrate":[126],"each":[127],"pitfall,":[128],"include":[130],"a":[131],"combination":[132],"theoretical":[134],"argumentation":[135],"as":[136,138],"well":[137],"examples":[139],"from":[140],"real":[141],"company's":[142],"experiments.":[143],"While":[144],"clearly":[147],"value":[148],"evaluating":[150],"personalized":[151],"by":[153],"means":[154],"experiments,":[158],"bear":[164],"mind":[166],"while":[167],"testing.":[168],"With":[169],"aim":[173],"increase":[175],"experimenters'":[177],"awareness":[178],"leading":[180],"improved":[182],"quality":[183],"reliability":[185],"results.":[188]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}