{"id":"https://openalex.org/W3194981584","doi":"https://doi.org/10.1145/3447548.3470819","title":"Online Advertising Incrementality Testing And Experimentation","display_name":"Online Advertising Incrementality Testing And Experimentation","publication_year":2021,"publication_date":"2021-08-12","ids":{"openalex":"https://openalex.org/W3194981584","doi":"https://doi.org/10.1145/3447548.3470819","mag":"3194981584"},"language":"en","primary_location":{"id":"doi:10.1145/3447548.3470819","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3447548.3470819","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113466602","display_name":"Joel Barajas","orcid":null},"institutions":[{"id":"https://openalex.org/I1302485747","display_name":"Verizon (United States)","ror":"https://ror.org/02vdyxx64","country_code":"US","type":"company","lineage":["https://openalex.org/I1302485747"]},{"id":"https://openalex.org/I4210134091","display_name":"Yahoo (United States)","ror":"https://ror.org/040dkzz12","country_code":"US","type":"company","lineage":["https://openalex.org/I4210134091"]},{"id":"https://openalex.org/I4401726916","display_name":"Verizon Media (United States)","ror":"https://ror.org/05dgrnf47","country_code":null,"type":"company","lineage":["https://openalex.org/I4401726916"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joel Barajas","raw_affiliation_strings":["Yahoo Research & Verizon Media, Sunnyvale, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yahoo Research & Verizon Media, Sunnyvale, CA, USA","institution_ids":["https://openalex.org/I1302485747","https://openalex.org/I4210134091","https://openalex.org/I4401726916"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062807220","display_name":"Narayan Bhamidipati","orcid":null},"institutions":[{"id":"https://openalex.org/I1302485747","display_name":"Verizon (United States)","ror":"https://ror.org/02vdyxx64","country_code":"US","type":"company","lineage":["https://openalex.org/I1302485747"]},{"id":"https://openalex.org/I4210134091","display_name":"Yahoo (United States)","ror":"https://ror.org/040dkzz12","country_code":"US","type":"company","lineage":["https://openalex.org/I4210134091"]},{"id":"https://openalex.org/I4401726916","display_name":"Verizon Media (United States)","ror":"https://ror.org/05dgrnf47","country_code":null,"type":"company","lineage":["https://openalex.org/I4401726916"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Narayan Bhamidipati","raw_affiliation_strings":["Yahoo Research & Verizon Media, Sunnyvale, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yahoo Research & Verizon Media, Sunnyvale, CA, USA","institution_ids":["https://openalex.org/I1302485747","https://openalex.org/I4210134091","https://openalex.org/I4401726916"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5113808513","display_name":"James G. Shanahan","orcid":null},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James G. Shanahan","raw_affiliation_strings":["Church and Duncan Group Inc, UC Berkeley, San Francisco, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Church and Duncan Group Inc, UC Berkeley, San Francisco, CA, USA","institution_ids":["https://openalex.org/I95457486"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4027","last_page":"4028"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9818000197410583,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6441026926040649},{"id":"https://openalex.org/keywords/online-advertising","display_name":"Online advertising","score":0.5930848717689514},{"id":"https://openalex.org/keywords/personalization","display_name":"Personalization","score":0.5773599147796631},{"id":"https://openalex.org/keywords/operationalization","display_name":"Operationalization","score":0.5518056154251099},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4458490312099457},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.28443431854248047},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.2065998613834381}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6441026926040649},{"id":"https://openalex.org/C512338625","wikidata":"https://www.wikidata.org/wiki/Q624902","display_name":"Online advertising","level":3,"score":0.5930848717689514},{"id":"https://openalex.org/C183003079","wikidata":"https://www.wikidata.org/wiki/Q1000371","display_name":"Personalization","level":2,"score":0.5773599147796631},{"id":"https://openalex.org/C9354725","wikidata":"https://www.wikidata.org/wiki/Q286017","display_name":"Operationalization","level":2,"score":0.5518056154251099},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4458490312099457},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.28443431854248047},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.2065998613834381},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3447548.3470819","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3447548.3470819","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure","score":0.6100000143051147}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":9,"referenced_works":["https://openalex.org/W1516659296","https://openalex.org/W1921202903","https://openalex.org/W2346728723","https://openalex.org/W2564917231","https://openalex.org/W3011423189","https://openalex.org/W3094241458","https://openalex.org/W3122414560","https://openalex.org/W3125057276","https://openalex.org/W3156050170"],"related_works":["https://openalex.org/W2108595774","https://openalex.org/W2486082319","https://openalex.org/W3095059880","https://openalex.org/W2267509780","https://openalex.org/W2374116733","https://openalex.org/W4287761343","https://openalex.org/W2111300691","https://openalex.org/W2892218161","https://openalex.org/W1506282065","https://openalex.org/W87991986"],"abstract_inverted_index":{"Online":[0],"advertising":[1,73,176],"has":[2,75],"historically":[3],"been":[4,57],"approached":[5],"as":[6],"user":[7],"targeting":[8],"and":[9,21,96,109,119,140,162,178,214,228,247,257],"ad-to-user":[10],"matching":[11],"problems":[12],"within":[13,153],"sophisticated":[14],"optimization":[15],"algorithms.":[16],"As":[17],"the":[18,28,37,63,69,77,97,124,137,144,154,170,174,179,191,201,211,217,238,275],"research":[19,164],"area":[20],"ad":[22,98,258],"tech":[23,99],"industry":[24,100,177,239],"have":[25,34,56,101],"progressed":[26],"over":[27,274],"last":[29,276],"couple":[30,277],"of":[31,41,71,79,87,126,147,173,181,219,266,278],"decades,":[32],"advertisers":[33],"increasingly":[35],"emphasized":[36],"causal":[38],"effect":[39,114],"estimation":[40],"their":[42],"ads":[43],"(aka":[44],"incrementality)":[45],"using":[46],"controlled":[47,83],"experiments":[48,84,267],"(or":[49,117],"A/B":[50],"testing).":[51],"Even":[52],"though":[53],"observational":[54],"approaches":[55],"derived":[58],"in":[59,91,136,200,210,216,237,252],"marketing":[60,92,155],"science":[61,156,161],"since":[62],"80s":[64],"including":[65],"media":[66],"mix":[67],"models,":[68],"availability":[70],"online":[72,94,175],"personalization":[74],"enabled":[76],"deployment":[78],"more":[80],"rigorous":[81],"randomized":[82],"with":[85],"millions":[86],"individuals.":[88],"These":[89],"evolutions":[90],"science,":[93],"advertising,":[95],"posed":[102],"incredible":[103],"challenges":[104,135,168],"for":[105,268],"engineers,":[106],"data":[107,160],"scientists,":[108],"marketers":[110],"alike.":[111],"With":[112],"low":[113],"percentage":[115],"differences":[116],"lift)":[118],"often":[120],"sparse":[121],"conversion":[122],"rates,":[123],"development":[125,218],"incrementality":[127,202,220],"testing":[128,203,221,224,235],"platforms":[129],"at":[130],"scale":[131],"suggests":[132],"tremendous":[133],"engineering":[134],"measurement":[138],"precision":[139],"detailed":[141],"implementation.":[142],"Similarly,":[143],"correct":[145],"interpretation":[146],"results":[148,232],"addressing":[149],"a":[150,196,250,253],"business":[151,207],"goal":[152],"domain":[157],"requires":[158],"significant":[159],"experimentation":[163],"expertise.":[165],"All":[166],"these":[167,260],"on":[169,245,263],"ongoing":[171],"evolution":[172,236],"heterogeneity":[180],"its":[182],"sources":[183],"(social,":[184],"paid":[185],"search,":[186],"native,":[187],"programmatic,":[188],"etc).":[189],"In":[190],"current":[192],"tutorial,":[193],"we":[194],"propose":[195],"practical,":[197],"grounded":[198],"view":[199],"landscape,":[204],"including:":[205],"The":[206,223],"need":[208],"Solutions":[209],"literature":[212],"Design":[213],"choices":[215],"platform":[222,251],"cycle,":[225],"case":[226],"studies,":[227],"recommendations":[229],"to":[230,270],"effective":[231],"delivery":[233],"Incrementality":[234],"We":[240],"will":[241],"provide":[242],"first-hand":[243],"lessons":[244],"developing":[246],"operationalizing":[248],"such":[249],"major":[254],"combined":[255],"DSP":[256],"network;":[259],"are":[261],"based":[262],"running":[264],"tens":[265],"up":[269],"two":[271],"months":[272],"each":[273],"years.":[279]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}