{"id":"https://openalex.org/W3166749140","doi":"https://doi.org/10.1145/3447548.3467091","title":"Network Experimentation at Scale","display_name":"Network Experimentation at Scale","publication_year":2021,"publication_date":"2021-08-12","ids":{"openalex":"https://openalex.org/W3166749140","doi":"https://doi.org/10.1145/3447548.3467091","mag":"3166749140"},"language":"en","primary_location":{"id":"doi:10.1145/3447548.3467091","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3447548.3467091","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3447548.3467091","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3447548.3467091","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5038907218","display_name":"Brian Karrer","orcid":"https://orcid.org/0000-0002-6252-8332"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Brian Karrer","raw_affiliation_strings":["Facebook, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444","https://openalex.org/I4210099336"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034452669","display_name":"Liang Shi","orcid":"https://orcid.org/0000-0002-6585-6657"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liang Shi","raw_affiliation_strings":["Facebook, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444","https://openalex.org/I4210099336"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009130093","display_name":"Monica Bhole","orcid":null},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Monica Bhole","raw_affiliation_strings":["Facebook, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444","https://openalex.org/I4210099336"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086971492","display_name":"Matt Goldman","orcid":null},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matt Goldman","raw_affiliation_strings":["Facebook, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444","https://openalex.org/I4210099336"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002757499","display_name":"Tyrone Palmer","orcid":null},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tyrone Palmer","raw_affiliation_strings":["Facebook, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444","https://openalex.org/I4210099336"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023807975","display_name":"Charlie Gelman","orcid":null},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Charlie Gelman","raw_affiliation_strings":["Facebook, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444","https://openalex.org/I4210099336"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046427713","display_name":"Mikael Konutgan","orcid":null},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mikael Konutgan","raw_affiliation_strings":["Facebook, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444","https://openalex.org/I4210099336"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103139435","display_name":"Feng Sun","orcid":"https://orcid.org/0000-0002-8012-2993"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Feng Sun","raw_affiliation_strings":["Facebook, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444","https://openalex.org/I4210099336"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5038907218"],"corresponding_institution_ids":["https://openalex.org/I4210099336","https://openalex.org/I4210114444"],"apc_list":null,"apc_paid":null,"fwci":2.8359,"has_fulltext":true,"cited_by_count":18,"citation_normalized_percentile":{"value":0.91947547,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"3106","last_page":"3116"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11539","display_name":"Survey Methodology and Nonresponse","score":0.9657999873161316,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10557","display_name":"Social Media and Politics","score":0.9632999897003174,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.746300458908081},{"id":"https://openalex.org/keywords/interference","display_name":"Interference (communication)","score":0.7439383268356323},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.6479194164276123},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5758785009384155},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.5481389760971069},{"id":"https://openalex.org/keywords/regression","display_name":"Regression","score":0.5254449248313904},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.45756796002388},{"id":"https://openalex.org/keywords/cluster","display_name":"Cluster (spacecraft)","score":0.43874645233154297},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3339247703552246},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3244974911212921},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.18454939126968384},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.11750265955924988},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09395593404769897}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.746300458908081},{"id":"https://openalex.org/C32022120","wikidata":"https://www.wikidata.org/wiki/Q797225","display_name":"Interference (communication)","level":3,"score":0.7439383268356323},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.6479194164276123},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5758785009384155},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5481389760971069},{"id":"https://openalex.org/C83546350","wikidata":"https://www.wikidata.org/wiki/Q1139051","display_name":"Regression","level":2,"score":0.5254449248313904},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.45756796002388},{"id":"https://openalex.org/C164866538","wikidata":"https://www.wikidata.org/wiki/Q367351","display_name":"Cluster (spacecraft)","level":2,"score":0.43874645233154297},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3339247703552246},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3244974911212921},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.18454939126968384},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.11750265955924988},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09395593404769897},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3447548.3467091","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3447548.3467091","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3447548.3467091","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3447548.3467091","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3447548.3467091","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3447548.3467091","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3166749140.pdf","grobid_xml":"https://content.openalex.org/works/W3166749140.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W155390068","https://openalex.org/W1550914445","https://openalex.org/W2012797835","https://openalex.org/W2063910719","https://openalex.org/W2066598607","https://openalex.org/W2076731491","https://openalex.org/W2080770071","https://openalex.org/W2095293504","https://openalex.org/W2126002144","https://openalex.org/W2131681506","https://openalex.org/W2132917208","https://openalex.org/W2152886806","https://openalex.org/W2169068368","https://openalex.org/W2314734263","https://openalex.org/W2509295096","https://openalex.org/W2743673655","https://openalex.org/W2918781801","https://openalex.org/W2953117652","https://openalex.org/W2963351205","https://openalex.org/W2964213576","https://openalex.org/W2964337893","https://openalex.org/W2977410870","https://openalex.org/W3099125911","https://openalex.org/W3099768174","https://openalex.org/W3106294663","https://openalex.org/W3123609002"],"related_works":["https://openalex.org/W2375786911","https://openalex.org/W2367630196","https://openalex.org/W756944427","https://openalex.org/W2349547031","https://openalex.org/W2358871510","https://openalex.org/W278619663","https://openalex.org/W2107962325","https://openalex.org/W4385061335","https://openalex.org/W2000663367","https://openalex.org/W2585070004"],"abstract_inverted_index":{"We":[0,16,130],"describe":[1],"our":[2,44,72],"network":[3,142],"experimentation":[4,150],"framework,":[5],"deployed":[6],"at":[7,41],"Facebook,":[8],"which":[9],"accounts":[10],"for":[11,57,69,83],"interference":[12,84,126],"between":[13],"experimental":[14],"units.":[15],"document":[17],"this":[18,39,132,149],"system,":[19],"including":[20],"the":[21,33,146],"design":[22],"and":[23,26,144],"estimation":[24,45],"procedures,":[25],"detail":[27],"insights":[28],"we":[29,47,75,93],"have":[30,37],"gained":[31],"from":[32,122],"many":[34],"experiments":[35,124],"that":[36,53,77,95,139],"used":[38],"system":[40],"scale.":[42],"In":[43,91],"procedure,":[46],"introduce":[48],"a":[49,65,99,109],"cluster-based":[50],"regression":[51,73],"adjustment":[52],"substantially":[54],"improves":[55],"precision":[56],"estimating":[58],"global":[59],"treatment":[60,100],"effects,":[61],"as":[62,64],"well":[63],"procedure":[66],"to":[67,98],"test":[68],"interference.":[70],"With":[71],"adjustment,":[74],"find":[76],"imbalanced":[78],"clusters":[79,87],"can":[80,101],"better":[81],"account":[82],"than":[85],"balanced":[86],"without":[88],"sacrificing":[89],"accuracy.":[90],"addition,":[92],"show":[94],"logging":[96],"exposure":[97],"result":[102],"in":[103,113,127],"additional":[104],"variance":[105],"reduction.":[106],"Interference":[107],"is":[108,119],"widely":[110],"acknowledged":[111],"issue":[112],"online":[114,128],"field":[115],"experiments,":[116],"yet":[117],"there":[118],"less":[120],"evidence":[121],"real-world":[123],"demonstrating":[125],"settings.":[129],"fill":[131],"gap":[133],"by":[134],"describing":[135],"two":[136],"case":[137],"studies":[138],"capture":[140],"significant":[141],"effects":[143],"highlight":[145],"value":[147],"of":[148],"framework.":[151]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1}],"updated_date":"2026-03-09T08:58:05.943551","created_date":"2025-10-10T00:00:00"}