{"id":"https://openalex.org/W3117329100","doi":"https://doi.org/10.1145/3437963.3441742","title":"How to Measure Your App: A Couple of Pitfalls and Remedies in Measuring App Performance in Online Controlled Experiments","display_name":"How to Measure Your App: A Couple of Pitfalls and Remedies in Measuring App Performance in Online Controlled Experiments","publication_year":2021,"publication_date":"2021-03-06","ids":{"openalex":"https://openalex.org/W3117329100","doi":"https://doi.org/10.1145/3437963.3441742","mag":"3117329100"},"language":"en","primary_location":{"id":"doi:10.1145/3437963.3441742","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3437963.3441742","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 14th ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103945494","display_name":"Yuxiang Xie","orcid":null},"institutions":[{"id":"https://openalex.org/I4210142583","display_name":"Snap (United States)","ror":"https://ror.org/04dgkhg68","country_code":"US","type":"company","lineage":["https://openalex.org/I4210142583"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Yuxiang Xie","raw_affiliation_strings":["Snap Inc., Santa Monica, CA, USA"],"affiliations":[{"raw_affiliation_string":"Snap Inc., Santa Monica, CA, USA","institution_ids":["https://openalex.org/I4210142583"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003470834","display_name":"Meng Xu","orcid":"https://orcid.org/0000-0002-3634-0547"},"institutions":[{"id":"https://openalex.org/I4210142583","display_name":"Snap (United States)","ror":"https://ror.org/04dgkhg68","country_code":"US","type":"company","lineage":["https://openalex.org/I4210142583"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Meng Xu","raw_affiliation_strings":["Snap Inc., Santa Monica, CA, USA"],"affiliations":[{"raw_affiliation_string":"Snap Inc., Santa Monica, CA, USA","institution_ids":["https://openalex.org/I4210142583"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003045964","display_name":"Evan Chow","orcid":null},"institutions":[{"id":"https://openalex.org/I4210142583","display_name":"Snap (United States)","ror":"https://ror.org/04dgkhg68","country_code":"US","type":"company","lineage":["https://openalex.org/I4210142583"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Evan Chow","raw_affiliation_strings":["Snap Inc., Santa Monica, CA, USA"],"affiliations":[{"raw_affiliation_string":"Snap Inc., Santa Monica, CA, USA","institution_ids":["https://openalex.org/I4210142583"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101666398","display_name":"Xiaolin Shi","orcid":"https://orcid.org/0000-0002-3705-1552"},"institutions":[{"id":"https://openalex.org/I4210142583","display_name":"Snap (United States)","ror":"https://ror.org/04dgkhg68","country_code":"US","type":"company","lineage":["https://openalex.org/I4210142583"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaolin Shi","raw_affiliation_strings":["Snap Inc., Santa Monica, CA, USA"],"affiliations":[{"raw_affiliation_string":"Snap Inc., Santa Monica, CA, USA","institution_ids":["https://openalex.org/I4210142583"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5103945494"],"corresponding_institution_ids":["https://openalex.org/I4210142583"],"apc_list":null,"apc_paid":null,"fwci":0.464,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.63530755,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"949","last_page":"957"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7740598917007446},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5343571901321411},{"id":"https://openalex.org/keywords/performance-metric","display_name":"Performance metric","score":0.47504425048828125},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.4637923836708069},{"id":"https://openalex.org/keywords/mobile-apps","display_name":"Mobile apps","score":0.4335528314113617},{"id":"https://openalex.org/keywords/rss","display_name":"RSS","score":0.4331167936325073},{"id":"https://openalex.org/keywords/quantile","display_name":"Quantile","score":0.4174795150756836},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3658719062805176},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.35013359785079956},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3083667755126953},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.1251530647277832}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7740598917007446},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5343571901321411},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.47504425048828125},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.4637923836708069},{"id":"https://openalex.org/C2988145974","wikidata":"https://www.wikidata.org/wiki/Q620615","display_name":"Mobile apps","level":2,"score":0.4335528314113617},{"id":"https://openalex.org/C2385561","wikidata":"https://www.wikidata.org/wiki/Q45432","display_name":"RSS","level":2,"score":0.4331167936325073},{"id":"https://openalex.org/C118671147","wikidata":"https://www.wikidata.org/wiki/Q578714","display_name":"Quantile","level":2,"score":0.4174795150756836},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3658719062805176},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35013359785079956},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3083667755126953},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.1251530647277832},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3437963.3441742","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3437963.3441742","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 14th ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.6100000143051147,"id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":49,"referenced_works":["https://openalex.org/W105168457","https://openalex.org/W658722133","https://openalex.org/W1584855941","https://openalex.org/W1978108654","https://openalex.org/W1996657939","https://openalex.org/W1997753419","https://openalex.org/W2029941278","https://openalex.org/W2035846549","https://openalex.org/W2043851564","https://openalex.org/W2077355162","https://openalex.org/W2085816378","https://openalex.org/W2100264883","https://openalex.org/W2118502261","https://openalex.org/W2131184539","https://openalex.org/W2132324013","https://openalex.org/W2139122730","https://openalex.org/W2139577967","https://openalex.org/W2150291618","https://openalex.org/W2168458505","https://openalex.org/W2211188029","https://openalex.org/W2486117411","https://openalex.org/W2567202186","https://openalex.org/W2577227262","https://openalex.org/W2582220249","https://openalex.org/W2743849295","https://openalex.org/W2761665523","https://openalex.org/W2809468631","https://openalex.org/W2890543633","https://openalex.org/W2900414111","https://openalex.org/W2946387282","https://openalex.org/W2952127798","https://openalex.org/W2955212083","https://openalex.org/W2965222965","https://openalex.org/W2980495984","https://openalex.org/W2997919412","https://openalex.org/W3103282043","https://openalex.org/W3122781290","https://openalex.org/W3122812581","https://openalex.org/W3123935396","https://openalex.org/W3125459412","https://openalex.org/W4212774754","https://openalex.org/W4241561347","https://openalex.org/W4242484299","https://openalex.org/W4242720768","https://openalex.org/W4250274144","https://openalex.org/W4300187280","https://openalex.org/W6688406494","https://openalex.org/W6691501407","https://openalex.org/W6772013027"],"related_works":["https://openalex.org/W4361804730","https://openalex.org/W2142113611","https://openalex.org/W2334467465","https://openalex.org/W2087870008","https://openalex.org/W2162534555","https://openalex.org/W2752178021","https://openalex.org/W2107419853","https://openalex.org/W2143024819","https://openalex.org/W4247159817","https://openalex.org/W2964201926"],"abstract_inverted_index":{"Effectively":[0],"measuring,":[1],"understanding,":[2],"and":[3,44,69,131,165,168,196,198,214,218],"improving":[4],"mobile":[5,13,18,154,163],"app":[6,14,38,50],"performance":[7,31,39,62,81,88,120,129,152,193],"is":[8,77],"of":[9,30,103,118,150],"paramount":[10],"importance":[11],"for":[12,153,200],"developers.":[15],"Across":[16],"the":[17,169],"Internet":[19],"landscape,":[20],"companies":[21],"run":[22],"online":[23],"controlled":[24],"experiments":[25],"(A/B":[26],"tests)":[27],"with":[28,101,134],"thousands":[29,102],"metrics":[32,121,130],"in":[33,72,90,95,114,128,146,161],"order":[34],"to":[35,45,61,78,125],"understand":[36],"how":[37],"causally":[40],"impacts":[41],"user":[42,54,135,166,178],"retention":[43],"guard":[46],"against":[47],"service":[48],"or":[49,92],"regressions":[51],"that":[52,122],"degrade":[53],"experiences.":[55],"To":[56,181],"capture":[57],"certain":[58],"characteristics":[59],"particular":[60],"metrics,":[63],"such":[64],"as":[65,83],"enormous":[66],"observation":[67],"volume":[68],"high":[70],"skewness":[71],"distribution,":[73],"an":[74],"industry-standard":[75,116,148],"practice":[76,149],"construct":[79],"a":[80,84],"metric":[82,194,202],"quantile":[85],"over":[86],"all":[87],"events":[89],"control":[91],"treatment":[93],"buckets":[94],"A/B":[96,104,216],"tests.":[97],"In":[98,138],"our":[99],"experience":[100],"tests":[105],"provided":[106],"by":[107,176],"Snap,":[108],"we":[109,141,186],"have":[110,205,219],"discovered":[111],"some":[112],"pitfalls":[113,145],"this":[115,139,147],"way":[117],"calculating":[119],"can":[123],"lead":[124],"unexplained":[126],"movements":[127],"unexpected":[132],"misalignment":[133],"engagement":[136,179],"metrics.":[137],"paper,":[140],"discuss":[142],"two":[143,184],"major":[144],"measuring":[151],"apps.":[155],"One":[156],"arises":[157,171],"from":[158,172],"strong":[159],"heterogeneity":[160],"both":[162,211],"devices":[164],"engagement,":[167],"other":[170],"self-selection":[173],"bias":[174],"caused":[175],"post-treatment":[177],"changes.":[180],"remedy":[182],"these":[183,208],"pitfalls,":[185],"introduce":[187],"several":[188],"scalable":[189],"methods":[190,209],"including":[191],"user-level":[192],"calculation":[195],"imputation":[197],"matching":[199],"missing":[201],"values.":[203],"We":[204],"extensively":[206],"evaluated":[207],"on":[210],"simulation":[212],"data":[213],"real":[215],"tests,":[217],"deployed":[220],"them":[221],"into":[222],"Snap's":[223],"in-house":[224],"experimentation":[225],"platform.":[226]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}