{"id":"https://openalex.org/W2976331632","doi":"https://doi.org/10.1145/3360608","title":"PlanAlyzer: assessing threats to the validity of online experiments","display_name":"PlanAlyzer: assessing threats to the validity of online experiments","publication_year":2019,"publication_date":"2019-10-10","ids":{"openalex":"https://openalex.org/W2976331632","doi":"https://doi.org/10.1145/3360608","mag":"2976331632"},"language":"en","primary_location":{"id":"doi:10.1145/3360608","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3360608","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3360608","source":{"id":"https://openalex.org/S4210216081","display_name":"Proceedings of the ACM on Programming Languages","issn_l":"2475-1421","issn":["2475-1421"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Programming Languages","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3360608","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Emma Tosch","orcid":null},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Emma Tosch","raw_affiliation_strings":["University of Massachusetts Amherst, USA"],"affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, USA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Eytan Bakshy","orcid":null},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eytan Bakshy","raw_affiliation_strings":["Facebook, USA"],"affiliations":[{"raw_affiliation_string":"Facebook, USA","institution_ids":["https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Emery D. Berger","orcid":null},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Emery D. Berger","raw_affiliation_strings":["University of Massachusetts Amherst, USA"],"affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, USA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":null,"display_name":"David D. Jensen","orcid":null},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David D. Jensen","raw_affiliation_strings":["University of Massachusetts Amherst, USA"],"affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, USA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"last","author":{"id":null,"display_name":"J. Eliot B. Moss","orcid":null},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"J. Eliot B. Moss","raw_affiliation_strings":["University of Massachusetts Amherst, USA"],"affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, USA","institution_ids":["https://openalex.org/I24603500"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I24603500"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.13976904,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":95},"biblio":{"volume":"3","issue":"OOPSLA","first_page":"1","last_page":"30"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.7796000242233276,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.7796000242233276,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.039400000125169754,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.01549999974668026,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.5418000221252441},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5218999981880188},{"id":"https://openalex.org/keywords/internal-validity","display_name":"Internal validity","score":0.5148000121116638},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.4560999870300293},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4341999888420105},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.426800012588501},{"id":"https://openalex.org/keywords/causal-inference","display_name":"Causal inference","score":0.3785000145435333},{"id":"https://openalex.org/keywords/causal-model","display_name":"Causal model","score":0.37459999322891235}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7817000150680542},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.5418000221252441},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5218999981880188},{"id":"https://openalex.org/C51082289","wikidata":"https://www.wikidata.org/wiki/Q4113800","display_name":"Internal validity","level":2,"score":0.5148000121116638},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.4560999870300293},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4341999888420105},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.426800012588501},{"id":"https://openalex.org/C158600405","wikidata":"https://www.wikidata.org/wiki/Q5054566","display_name":"Causal inference","level":2,"score":0.3785000145435333},{"id":"https://openalex.org/C11671645","wikidata":"https://www.wikidata.org/wiki/Q5054567","display_name":"Causal model","level":2,"score":0.37459999322891235},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.3499000072479248},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3215000033378601},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3181000053882599},{"id":"https://openalex.org/C2778348673","wikidata":"https://www.wikidata.org/wiki/Q739302","display_name":"Production (economics)","level":2,"score":0.30869999527931213},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.30489999055862427},{"id":"https://openalex.org/C174106493","wikidata":"https://www.wikidata.org/wiki/Q1057880","display_name":"External validity","level":2,"score":0.2985000014305115},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.2976999878883362},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.29249998927116394},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.29249998927116394},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2888000011444092},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.28130000829696655},{"id":"https://openalex.org/C40305131","wikidata":"https://www.wikidata.org/wiki/Q2616305","display_name":"Obfuscation","level":2,"score":0.27489998936653137},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.2648000121116638},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.25060001015663147}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3360608","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3360608","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3360608","source":{"id":"https://openalex.org/S4210216081","display_name":"Proceedings of the ACM on Programming Languages","issn_l":"2475-1421","issn":["2475-1421"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Programming Languages","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1909.13649","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1909.13649","pdf_url":"https://arxiv.org/pdf/1909.13649","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3360608","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3360608","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3360608","source":{"id":"https://openalex.org/S4210216081","display_name":"Proceedings of the ACM on Programming Languages","issn_l":"2475-1421","issn":["2475-1421"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Programming Languages","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2976331632.pdf","grobid_xml":"https://content.openalex.org/works/W2976331632.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W1480909796","https://openalex.org/W1490576214","https://openalex.org/W1516659296","https://openalex.org/W1579695396","https://openalex.org/W1933103375","https://openalex.org/W1974561310","https://openalex.org/W1975566260","https://openalex.org/W1982205631","https://openalex.org/W1987993132","https://openalex.org/W2009489720","https://openalex.org/W2044123923","https://openalex.org/W2046536636","https://openalex.org/W2049389441","https://openalex.org/W2052681001","https://openalex.org/W2066787289","https://openalex.org/W2080982416","https://openalex.org/W2094873755","https://openalex.org/W2095056536","https://openalex.org/W2104942079","https://openalex.org/W2112508839","https://openalex.org/W2122049982","https://openalex.org/W2122067387","https://openalex.org/W2126908011","https://openalex.org/W2143891888","https://openalex.org/W2144344516","https://openalex.org/W2144572264","https://openalex.org/W2144849164","https://openalex.org/W2159928435","https://openalex.org/W2165612525","https://openalex.org/W2166000498","https://openalex.org/W2166730394","https://openalex.org/W2295266283","https://openalex.org/W2572804059","https://openalex.org/W2591456992","https://openalex.org/W2591716374","https://openalex.org/W2610974678","https://openalex.org/W3121797019","https://openalex.org/W3124658416"],"related_works":[],"abstract_inverted_index":{"Online":[0],"experiments":[1,19,46],"have":[2,34,131],"become":[3],"a":[4,113,121,133,142,166,188,209],"ubiquitous":[5],"aspect":[6],"of":[7,18,26,91,144,151,169,180,190,212,221,229],"design":[8,28,105],"and":[9,29,40,53,106,126,155,198,219,227],"engineering":[10],"processes":[11],"within":[12],"Internet":[13],"firms.":[14],"As":[15],"the":[16,24,82,88,178,224,230],"scale":[17],"has":[20,23,216],"grown,":[21],"so":[22],"complexity":[25],"their":[27,55],"implementation.":[30],"In":[31],"response,":[32],"firms":[33],"developed":[35],"software":[36],"frameworks":[37,49],"for":[38,85,141],"designing":[39],"deploying":[41],"online":[42,92],"experiments.":[43,93,129,182],"Ensuring":[44],"that":[45,54,101,119,137],"in":[47,77,103,194],"these":[48,181],"are":[50,57,96],"correctly":[51],"designed":[52],"results":[56,179],"trustworthy---referred":[58],"to":[59,124,146,162,172,203,206],"as":[60],"internal":[61,67,89,147],"validity---can":[62],"be":[63],"difficult.":[64],"Currently,":[65],"verifying":[66],"validity":[68,90,207],"requires":[69],"manual":[70],"inspection":[71],"by":[72],"someone":[73],"with":[74],"substantial":[75],"expertise":[76],"experimental":[78,104],"design.":[79],"We":[80,130,183],"present":[81],"first":[83],"approach":[84],"statically":[86],"checking":[87],"Our":[94,109],"checks":[95,138],"based":[97],"on":[98,187,208,223],"well-known":[99],"problems":[100],"arise":[102],"causal":[107,156],"inference.":[108],"analyses":[110,161,176],"target":[111],"PlanOut,":[112],"widely":[114],"deployed,":[115],"open-source":[116],"experimentation":[117],"framework":[118],"uses":[120,159],"domain-specific":[122],"language":[123],"specify":[125],"run":[127],"complex":[128],"built":[132],"tool":[134],"called":[135],"PlanAlyzer":[136,158,215],"PlanOut":[139,191],"programs":[140],"variety":[143],"threats":[145,205],"validity,":[148],"including":[149],"failures":[150],"randomization,":[152],"treatment":[153],"assignment,":[154],"sufficiency.":[157],"its":[160,201],"automatically":[163],"generate":[164],"contrasts,":[165],"key":[167],"type":[168],"information":[170],"required":[171],"perform":[173],"valid":[174],"statistical":[175],"over":[177],"demonstrate":[184],"PlanAlyzer's":[185],"utility":[186],"corpus":[189],"scripts":[192],"deployed":[193],"production":[195],"at":[196],"Facebook,":[197],"we":[199],"evaluate":[200],"ability":[202],"identify":[204],"mutated":[210,225],"subset":[211],"this":[213],"corpus.":[214],"both":[217],"precision":[218],"recall":[220],"92%":[222],"corpus,":[226],"82%":[228],"contrasts":[231],"it":[232],"generates":[233],"match":[234],"hand-specified":[235],"data.":[236]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2019-10-03T00:00:00"}