{"id":"https://openalex.org/W4416323730","doi":"https://doi.org/10.1109/access.2025.3634282","title":"Toward Reliable New-User Online Controlled Experiments: A Scientific Growth-Hacking Framework for Data-Driven Decision-Making","display_name":"Toward Reliable New-User Online Controlled Experiments: A Scientific Growth-Hacking Framework for Data-Driven Decision-Making","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416323730","doi":"https://doi.org/10.1109/access.2025.3634282"},"language":"en","primary_location":{"id":"doi:10.1109/access.2025.3634282","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3634282","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/access.2025.3634282","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Jian Tian","orcid":"https://orcid.org/0009-0005-8874-5801"},"institutions":[{"id":"https://openalex.org/I4210104818","display_name":"Scalable Network Technologies (United States)","ror":"https://ror.org/01jccpq86","country_code":"US","type":"company","lineage":["https://openalex.org/I4210104818"]},{"id":"https://openalex.org/I4210158589","display_name":"Endokrinologikum","ror":"https://ror.org/051nfce45","country_code":"DE","type":"healthcare","lineage":["https://openalex.org/I4210158589"]}],"countries":["DE","US"],"is_corresponding":false,"raw_author_name":"Jian Tian","raw_affiliation_strings":["TikTok Inc., Culver City, CA, USA","TikTok Inc, USA"],"raw_orcid":"https://orcid.org/0009-0005-8874-5801","affiliations":[{"raw_affiliation_string":"TikTok Inc., Culver City, CA, USA","institution_ids":["https://openalex.org/I4210104818"]},{"raw_affiliation_string":"TikTok Inc, USA","institution_ids":["https://openalex.org/I4210158589"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028139996","display_name":"Fengyuan Hu","orcid":"https://orcid.org/0000-0002-1756-7551"},"institutions":[{"id":"https://openalex.org/I4210104818","display_name":"Scalable Network Technologies (United States)","ror":"https://ror.org/01jccpq86","country_code":"US","type":"company","lineage":["https://openalex.org/I4210104818"]},{"id":"https://openalex.org/I4210158589","display_name":"Endokrinologikum","ror":"https://ror.org/051nfce45","country_code":"DE","type":"healthcare","lineage":["https://openalex.org/I4210158589"]}],"countries":["DE","US"],"is_corresponding":false,"raw_author_name":"Fan Hu","raw_affiliation_strings":["TikTok Inc., Culver City, CA, USA","TikTok Inc, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TikTok Inc., Culver City, CA, USA","institution_ids":["https://openalex.org/I4210104818"]},{"raw_affiliation_string":"TikTok Inc, USA","institution_ids":["https://openalex.org/I4210158589"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.30091449,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"13","issue":null,"first_page":"199833","last_page":"199845"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.4018999934196472,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.4018999934196472,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.15950000286102295,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13197","display_name":"Spreadsheets and End-User Computing","score":0.03539999946951866,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6715999841690063},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5224999785423279},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4733999967575073},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.4478999972343445},{"id":"https://openalex.org/keywords/type-i-and-type-ii-errors","display_name":"Type I and type II errors","score":0.3292999863624573},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.3237000107765198},{"id":"https://openalex.org/keywords/onboarding","display_name":"Onboarding","score":0.3176000118255615},{"id":"https://openalex.org/keywords/core","display_name":"Core (optical fiber)","score":0.3149000108242035}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.765500009059906},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6715999841690063},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5224999785423279},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4733999967575073},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.4478999972343445},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3865000009536743},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3853999972343445},{"id":"https://openalex.org/C40696583","wikidata":"https://www.wikidata.org/wiki/Q989120","display_name":"Type I and type II errors","level":2,"score":0.3292999863624573},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.3237000107765198},{"id":"https://openalex.org/C2779185108","wikidata":"https://www.wikidata.org/wiki/Q7091744","display_name":"Onboarding","level":2,"score":0.3176000118255615},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.3149000108242035},{"id":"https://openalex.org/C163985040","wikidata":"https://www.wikidata.org/wiki/Q1172399","display_name":"Data acquisition","level":2,"score":0.30630001425743103},{"id":"https://openalex.org/C96608239","wikidata":"https://www.wikidata.org/wiki/Q1199823","display_name":"Statistical power","level":2,"score":0.3003999888896942},{"id":"https://openalex.org/C80478641","wikidata":"https://www.wikidata.org/wiki/Q195771","display_name":"Sequential analysis","level":2,"score":0.28850001096725464},{"id":"https://openalex.org/C2777220311","wikidata":"https://www.wikidata.org/wiki/Q6423340","display_name":"Knowledge acquisition","level":2,"score":0.2881999909877777},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.2867000102996826},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.2849000096321106},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.273499995470047},{"id":"https://openalex.org/C103088060","wikidata":"https://www.wikidata.org/wiki/Q1062839","display_name":"Error detection and correction","level":2,"score":0.26829999685287476},{"id":"https://openalex.org/C64543145","wikidata":"https://www.wikidata.org/wiki/Q162942","display_name":"Intersection (aeronautics)","level":2,"score":0.259799987077713},{"id":"https://openalex.org/C196921405","wikidata":"https://www.wikidata.org/wiki/Q786431","display_name":"Online algorithm","level":2,"score":0.2556000053882599},{"id":"https://openalex.org/C122637931","wikidata":"https://www.wikidata.org/wiki/Q118084","display_name":"Unit (ring theory)","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2025.3634282","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3634282","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:a8cbb0d04a404a818d6c6c4c4e994af6","is_oa":true,"landing_page_url":"https://doaj.org/article/a8cbb0d04a404a818d6c6c4c4e994af6","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 13, Pp 199833-199845 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2025.3634282","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3634282","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Sustaining":[0],"user":[1],"growth":[2,12],"has":[3],"become":[4],"a":[5,15,105,250],"strategic":[6],"imperative":[7],"for":[8,51,256],"digital":[9],"platforms,":[10],"and":[11,22,54,89,113,125,165,175,181,209,237,248,253],"hacking":[13],"provides":[14],"data-driven":[16],"approach":[17],"that":[18,61,109,148,200],"integrates":[19,110],"analytics,":[20],"experimentation,":[21],"rapid":[23],"iteration.":[24],"Online":[25],"controlled":[26,260],"experiments":[27],"(OCEs)":[28],"form":[29],"the":[30,74,122,159,187,224],"scientific":[31],"core":[32],"of":[33],"this":[34],"process":[35],"by":[36,93,120,158,222],"transforming":[37],"behavioral":[38],"hypotheses":[39],"into":[40,179],"validated":[41,218],"learning":[42],"at":[43],"scale.":[44],"However,":[45],"existing":[46],"OCEs":[47,103],"implementations":[48],"remain":[49],"unreliable":[50],"new-user":[52,102,228,258],"acquisition":[53,95,164,180,236],"activation.":[55],"Cloud-side":[56],"randomization":[57,67,123],"introduces":[58],"first-launch":[59,133],"delays":[60],"bias":[62],"onboarding":[63],"behavior,":[64],"whereas":[65],"local-side":[66],"causes":[68],"group":[69],"switching":[70],"upon":[71],"reinstallation,":[72],"violating":[73],"Stable":[75],"Unit":[76],"Treatment":[77],"Value":[78],"Assumption":[79],"(SUTVA).":[80],"In":[81],"addition,":[82],"retention-only":[83],"evaluation":[84],"inflates":[85],"both":[86,235],"Type":[87,90,203],"I":[88,204],"II":[91],"errors":[92],"overlooking":[94],"effects.":[96,183],"This":[97],"paper":[98],"advances":[99],"toward":[100],"reliable":[101],"through":[104,143,194],"two-part":[106],"reliability":[107,112,117,140],"framework":[108,147,225],"assignment":[111,134,189],"analytical":[114,215],"reliability.":[115],"Assignment":[116],"is":[118,141,191,217],"achieved":[119,142],"separating":[121],"unit":[124,127],"analysis":[126,146],"(RU/AU),":[128],"which":[129],"enables":[130],"low-delay":[131],"(near-zero-delay)":[132],"while":[135],"preserving":[136],"inference":[137],"validity.":[138],"Analytical":[139],"an":[144],"entry-retention":[145],"uses":[149],"<italic":[150,154],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[151,155],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">DNU</i>":[152],"\u00b7":[153],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">RR</i>,":[156],"normalized":[157],"randomization-unit":[160],"count,":[161],"to":[162,172],"integrate":[163],"retention":[166,182],"dimensions,":[167],"quantify":[168],"their":[169],"joint":[170],"contribution":[171],"DAU":[173],"growth,":[174],"decompose":[176],"observed":[177],"improvements":[178,233],"The":[184,212],"first":[185],"part,":[186,214],"framework\u2019s":[188],"reliability,":[190,216],"statistically":[192,231,251],"verified":[193],"more":[195],"than":[196],"1,000":[197],"AA":[198],"simulations":[199],"confirm":[201],"correct":[202],"error":[205],"control,":[206],"normal":[207],"t-statistics,":[208],"uniform":[210],"p-values.":[211],"second":[213],"in":[219,234],"production":[220],"traffic":[221],"deploying":[223],"with":[226],"real":[227],"strategies,":[229],"producing":[230],"significant":[232],"retention.":[238],"Together,":[239],"these":[240],"results":[241],"restore":[242],"causal":[243],"validity,":[244],"reduce":[245],"decision":[246],"errors,":[247],"provide":[249],"rigorous":[252],"interpretable":[254],"foundation":[255],"trustworthy":[257],"online":[259],"experiments.":[261]},"counts_by_year":[],"updated_date":"2025-12-03T23:09:05.601824","created_date":"2025-11-18T00:00:00"}