{"id":"https://openalex.org/W2950819002","doi":"https://doi.org/10.1145/3325426.3330287","title":"How Notification Queues Contain Hidden Natural Experiments","display_name":"How Notification Queues Contain Hidden Natural Experiments","publication_year":2019,"publication_date":"2019-06-12","ids":{"openalex":"https://openalex.org/W2950819002","doi":"https://doi.org/10.1145/3325426.3330287","mag":"2950819002"},"language":"en","primary_location":{"id":"doi:10.1145/3325426.3330287","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3325426.3330287","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 5th ACM Workshop on Mobile Systems for Computational Social Science","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063733773","display_name":"Craig Tutterow","orcid":null},"institutions":[{"id":"https://openalex.org/I1316064682","display_name":"LinkedIn (United States)","ror":"https://ror.org/02fyxhe35","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I1316064682"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Craig Tutterow","raw_affiliation_strings":["LinkedIn Corp., Sunnyvale, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LinkedIn Corp., Sunnyvale, CA, USA","institution_ids":["https://openalex.org/I1316064682"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064552332","display_name":"Guillaume Saint-Jacques","orcid":null},"institutions":[{"id":"https://openalex.org/I1316064682","display_name":"LinkedIn (United States)","ror":"https://ror.org/02fyxhe35","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I1316064682"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Guillaume Saint-Jacques","raw_affiliation_strings":["LinkedIn Corp., Sunnyvale, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LinkedIn Corp., Sunnyvale, CA, USA","institution_ids":["https://openalex.org/I1316064682"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1316064682"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.07975078,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"15","last_page":"20"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10646","display_name":"Experimental Behavioral Economics Studies","score":0.987500011920929,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11031","display_name":"Game Theory and Applications","score":0.986299991607666,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/natural-experiment","display_name":"Natural experiment","score":0.6550701260566711},{"id":"https://openalex.org/keywords/counterfactual-conditional","display_name":"Counterfactual conditional","score":0.6523902416229248},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6455366015434265},{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.6084004640579224},{"id":"https://openalex.org/keywords/queue","display_name":"Queue","score":0.5893141031265259},{"id":"https://openalex.org/keywords/causal-inference","display_name":"Causal inference","score":0.5664806962013245},{"id":"https://openalex.org/keywords/observational-study","display_name":"Observational study","score":0.5597923398017883},{"id":"https://openalex.org/keywords/queueing-theory","display_name":"Queueing theory","score":0.5210915803909302},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.5083703398704529},{"id":"https://openalex.org/keywords/randomized-experiment","display_name":"Randomized experiment","score":0.47872066497802734},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.32366690039634705},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.29986488819122314},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.17023316025733948},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1573607623577118},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.12275466322898865},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10384610295295715}],"concepts":[{"id":"https://openalex.org/C49630185","wikidata":"https://www.wikidata.org/wiki/Q6980675","display_name":"Natural experiment","level":2,"score":0.6550701260566711},{"id":"https://openalex.org/C71889745","wikidata":"https://www.wikidata.org/wiki/Q1783264","display_name":"Counterfactual conditional","level":3,"score":0.6523902416229248},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6455366015434265},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.6084004640579224},{"id":"https://openalex.org/C160403385","wikidata":"https://www.wikidata.org/wiki/Q220543","display_name":"Queue","level":2,"score":0.5893141031265259},{"id":"https://openalex.org/C158600405","wikidata":"https://www.wikidata.org/wiki/Q5054566","display_name":"Causal inference","level":2,"score":0.5664806962013245},{"id":"https://openalex.org/C23131810","wikidata":"https://www.wikidata.org/wiki/Q818574","display_name":"Observational study","level":2,"score":0.5597923398017883},{"id":"https://openalex.org/C22684755","wikidata":"https://www.wikidata.org/wiki/Q847526","display_name":"Queueing theory","level":2,"score":0.5210915803909302},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.5083703398704529},{"id":"https://openalex.org/C155108698","wikidata":"https://www.wikidata.org/wiki/Q1231081","display_name":"Randomized experiment","level":2,"score":0.47872066497802734},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32366690039634705},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.29986488819122314},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.17023316025733948},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1573607623577118},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.12275466322898865},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10384610295295715},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3325426.3330287","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3325426.3330287","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 5th ACM Workshop on Mobile Systems for Computational Social Science","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.4699999988079071}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W1590379693","https://openalex.org/W1968380849","https://openalex.org/W1986926921","https://openalex.org/W2003741412","https://openalex.org/W2019108039","https://openalex.org/W2055235810","https://openalex.org/W2063597719","https://openalex.org/W2107295826","https://openalex.org/W2121594732","https://openalex.org/W2141951594","https://openalex.org/W2149084727","https://openalex.org/W2149910108","https://openalex.org/W2152886806","https://openalex.org/W2156467326","https://openalex.org/W2463347236","https://openalex.org/W2743673655","https://openalex.org/W2806125611","https://openalex.org/W2944771437","https://openalex.org/W2951409510","https://openalex.org/W3037090540","https://openalex.org/W3043736046","https://openalex.org/W3122504924","https://openalex.org/W3123059484","https://openalex.org/W3144040573","https://openalex.org/W4242087913"],"related_works":["https://openalex.org/W2056582926","https://openalex.org/W3137864021","https://openalex.org/W2162910442","https://openalex.org/W2079879923","https://openalex.org/W4200271736","https://openalex.org/W3017854570","https://openalex.org/W2104420793","https://openalex.org/W2028689793","https://openalex.org/W4313936361","https://openalex.org/W4242448314"],"abstract_inverted_index":{"Randomized":[0],"experiments,":[1],"or":[2,41,51,58,71,248],"A/B":[3],"tests":[4],"are":[5,28],"used":[6,132],"to":[7,31,43,48,54,76,133,145,165,188,227,234,256,275],"estimate":[8,166],"the":[9,16,55,62,87,96,102,109,118,155,160,167,176,228,240,249,271,277],"causal":[10,78,168,243],"impact":[11],"of":[12,18,60,64,80,89,101,112,162,171,178,230,242,266,280],"a":[13,65,128,151,172,179,189,196,203,263],"feature":[14,258],"on":[15,154,159,175,183],"behavior":[17],"users":[19],"by":[20,219],"creating":[21],"two":[22],"parallel":[23],"universes":[24],"in":[25,86,138,270],"which":[26,139],"members":[27],"simultaneously":[29],"assigned":[30],"treatment":[32],"and":[33,99],"control.":[34],"However,":[35],"it":[36],"is":[37,143],"not":[38],"always":[39],"feasible":[40],"desirable":[42],"run":[44],"an":[45],"experiment":[46,153],"due":[47],"engineering":[49,97,130,250],"costs,":[50],"concerns":[52,104],"related":[53],"user":[56],"experience":[57],"ethics":[59],"randomizing":[61],"presence":[63],"feature.":[66],"Naturally":[67],"occurring":[68,237],"exogenous":[69,141],"variation,":[70],"'natural":[72],"experiments,'":[73],"allow":[74],"researchers":[75],"recover":[77],"estimates":[79],"peer":[81,169],"effects":[82,244],"from":[83,198],"observational":[84,210],"data":[85,268],"absence":[88],"experimental":[90],"manipulation.":[91],"Natural":[92],"experiments":[93],"trade":[94],"off":[95],"costs":[98,111,120,251],"some":[100,208],"ethical":[103],"associated":[105,121,252],"with":[106,108,122,253],"randomization":[107,255],"search":[110,119],"finding":[113],"instrumental":[114],"variables.":[115],"To":[116],"mitigate":[117],"discovering":[123],"natural":[124,140,152],"counterfactuals,":[125],"we":[126],"identify":[127,150],"common":[129],"requirement":[131],"scale":[134],"massive":[135],"online":[136,254],"systems,":[137],"variation":[142],"likely":[144],"exist:":[146],"notification":[147,163,232],"queuing.":[148],"We":[149,192],"LinkedIn":[156],"platform":[157],"based":[158],"order":[161],"queues":[164,233],"effect":[170,218],"received":[173],"message":[174,180,197],"engagement":[177],"recipient":[181],"(based":[182],"work":[184],"anniversary":[185],"announcements":[186],"distributed":[187],"member's":[190,204],"connections).":[191],"show":[193],"that":[194,207],"receiving":[195],"another":[199],"member":[200],"significantly":[201],"increases":[202],"engagement,":[205],"but":[206],"popular":[209],"specifications,":[211],"such":[212],"as":[213,220,222],"fixed-effects":[214],"estimators,":[215],"overestimate":[216],"this":[217],"much":[221],"2.7x.":[223],"The":[224],"study":[225],"points":[226],"benefits":[229,279],"using":[231],"discover":[235],"naturally":[236],"counterfactuals":[238],"for":[239],"estimation":[241],"without":[245],"experimenter":[246],"intervention":[247],"different":[257],"sets.":[259],"It":[260],"also":[261],"implies":[262],"potential":[264],"benefit":[265],"involving":[267],"scientists":[269],"system":[272],"design":[273],"process":[274],"maximize":[276],"informational":[278],"software":[281],"systems.":[282]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}