{"id":"https://openalex.org/W3131754018","doi":"https://doi.org/10.1080/00401706.2022.2124309","title":"Novelty and Primacy: A Long-Term Estimator for Online Experiments","display_name":"Novelty and Primacy: A Long-Term Estimator for Online Experiments","publication_year":2022,"publication_date":"2022-09-23","ids":{"openalex":"https://openalex.org/W3131754018","doi":"https://doi.org/10.1080/00401706.2022.2124309","mag":"3131754018"},"language":"en","primary_location":{"id":"doi:10.1080/00401706.2022.2124309","is_oa":false,"landing_page_url":"https://doi.org/10.1080/00401706.2022.2124309","pdf_url":null,"source":{"id":"https://openalex.org/S985303","display_name":"Technometrics","issn_l":"0040-1706","issn":["0040-1706","1537-2723"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547"],"host_organization_lineage_names":["Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Technometrics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5022955397","display_name":"Soheil Sadeghi","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Soheil Sadeghi","raw_affiliation_strings":["Microsoft, Redmond, WA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039548707","display_name":"Somit Gupta","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Somit Gupta","raw_affiliation_strings":["Microsoft, Redmond, WA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089151145","display_name":"Stefan Gramatovici","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stefan Gramatovici","raw_affiliation_strings":["Microsoft, Redmond, WA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069822660","display_name":"Jiannan Lu","orcid":"https://orcid.org/0000-0002-8839-6024"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiannan Lu","raw_affiliation_strings":["Microsoft, Redmond, WA"],"raw_orcid":"https://orcid.org/0000-0002-8839-6024","affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103228598","display_name":"Hao Ai","orcid":"https://orcid.org/0000-0003-2104-3352"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hao Ai","raw_affiliation_strings":["Microsoft, Redmond, WA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061139804","display_name":"Ruhan Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ruhan Zhang","raw_affiliation_strings":["Microsoft, Redmond, WA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5022955397"],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":3.7249,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.94245637,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":"64","issue":"4","first_page":"524","last_page":"534"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9506999850273132,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9089000225067139,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/novelty","display_name":"Novelty","score":0.9369522929191589},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6682052612304688},{"id":"https://openalex.org/keywords/term","display_name":"Term (time)","score":0.5554917454719543},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.43886011838912964},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.42279481887817383},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3596656620502472},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.35853683948516846},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.355348140001297},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.19995161890983582},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.10358867049217224}],"concepts":[{"id":"https://openalex.org/C2778738651","wikidata":"https://www.wikidata.org/wiki/Q16546687","display_name":"Novelty","level":2,"score":0.9369522929191589},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6682052612304688},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.5554917454719543},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.43886011838912964},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.42279481887817383},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3596656620502472},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.35853683948516846},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.355348140001297},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.19995161890983582},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.10358867049217224},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1080/00401706.2022.2124309","is_oa":false,"landing_page_url":"https://doi.org/10.1080/00401706.2022.2124309","pdf_url":null,"source":{"id":"https://openalex.org/S985303","display_name":"Technometrics","issn_l":"0040-1706","issn":["0040-1706","1537-2723"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547"],"host_organization_lineage_names":["Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Technometrics","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.6200000047683716,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":44,"referenced_works":["https://openalex.org/W166389007","https://openalex.org/W1575232436","https://openalex.org/W1975566260","https://openalex.org/W1982544994","https://openalex.org/W1998653759","https://openalex.org/W1999147675","https://openalex.org/W2012646589","https://openalex.org/W2051081490","https://openalex.org/W2074380434","https://openalex.org/W2077319930","https://openalex.org/W2088247534","https://openalex.org/W2097636528","https://openalex.org/W2097882302","https://openalex.org/W2110228583","https://openalex.org/W2117859136","https://openalex.org/W2122679016","https://openalex.org/W2130527179","https://openalex.org/W2138572614","https://openalex.org/W2154062944","https://openalex.org/W2158125716","https://openalex.org/W2167067970","https://openalex.org/W2169113736","https://openalex.org/W2170832294","https://openalex.org/W2468478068","https://openalex.org/W2509860763","https://openalex.org/W2512464541","https://openalex.org/W2542081672","https://openalex.org/W2584778415","https://openalex.org/W2743849295","https://openalex.org/W2765165237","https://openalex.org/W2768977818","https://openalex.org/W2908235130","https://openalex.org/W2945720420","https://openalex.org/W2952127798","https://openalex.org/W2955212083","https://openalex.org/W2968595188","https://openalex.org/W2987812053","https://openalex.org/W2995790814","https://openalex.org/W3011423189","https://openalex.org/W3125512578","https://openalex.org/W3150893739","https://openalex.org/W4296159956","https://openalex.org/W4300879629","https://openalex.org/W6606778417"],"related_works":["https://openalex.org/W2381242807","https://openalex.org/W2751208925","https://openalex.org/W2347541121","https://openalex.org/W4288804799","https://openalex.org/W3089617106","https://openalex.org/W3032237421","https://openalex.org/W2080951048","https://openalex.org/W3011883280","https://openalex.org/W2390346111","https://openalex.org/W2369082698"],"abstract_inverted_index":{"Online":[0],"experiments":[1,19,156],"are":[2,20,28,41],"the":[3,72,86,97,164],"gold":[4],"standard":[5],"for":[6,44,108],"evaluating":[7],"impact":[8],"on":[9,57,137],"user":[10,126],"experience":[11],"and":[12,124,151,178,182],"accelerating":[13],"innovation":[14],"in":[15,23,154,172,187],"software.":[16],"However,":[17],"since":[18],"typically":[21],"limited":[22],"duration,":[24],"observed":[25],"treatment":[26,46,193],"effects":[27],"not":[29],"always":[30],"stable,":[31],"sometimes":[32],"revealing":[33],"increasing":[34],"or":[35,68],"decreasing":[36],"patterns":[37],"over":[38,50,82],"time.":[39,51,83,196],"There":[40],"multiple":[42],"causes":[43],"a":[45,58,92,118],"effect":[47],"to":[48,74,80,112,140,149,168,183],"change":[49],"In":[52,128],"this":[53,129,147],"article,":[54,130],"we":[55,131],"focus":[56],"particular":[59],"cause,":[60],"user-learning,":[61],"which":[62],"is":[63,101],"primarily":[64],"associated":[65],"with":[66,89,163,195],"novelty":[67],"primacy.":[69],"Novelty":[70],"describes":[71,85],"desire":[73],"use":[75,146,177],"new":[76],"technology":[77,90],"that":[78],"tends":[79],"diminish":[81],"Primacy":[84],"growing":[87],"engagement":[88],"as":[91],"result":[93],"of":[94,96,121,174,176,189,192],"adoption":[95],"innovation.":[98],"Estimating":[99],"user-learning":[100,142,153],"critical":[102],"because":[103],"it":[104],"holds":[105],"experimentation":[106],"responsible":[107],"trustworthiness,":[109],"empowers":[110],"organizations":[111],"make":[113],"better":[114],"decisions":[115],"by":[116],"providing":[117],"long-term":[119],"view":[120],"expected":[122],"impact,":[123],"prevents":[125],"dissatisfaction.":[127],"propose":[132],"an":[133],"observational":[134],"approach,":[135],"based":[136],"difference-in-differences":[138],"technique":[139],"estimate":[141,152],"at":[143,157],"scale.":[144],"We":[145,159],"approach":[148,162],"test":[150],"many":[155],"Microsoft.":[158],"compare":[160],"our":[161],"existing":[165],"experimental":[166],"method":[167],"show":[169],"its":[170,185],"benefits":[171],"terms":[173],"ease":[175],"higher":[179],"statistical":[180],"power,":[181],"discuss":[184],"limitation":[186],"presence":[188],"other":[190],"forms":[191],"interaction":[194]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":5}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}