{"id":"https://openalex.org/W1923283532","doi":"https://doi.org/10.1109/icdm.2015.32","title":"Online Model Evaluation in a Large-Scale Computational Advertising Platform","display_name":"Online Model Evaluation in a Large-Scale Computational Advertising Platform","publication_year":2015,"publication_date":"2015-11-01","ids":{"openalex":"https://openalex.org/W1923283532","doi":"https://doi.org/10.1109/icdm.2015.32","mag":"1923283532"},"language":"en","primary_location":{"id":"doi:10.1109/icdm.2015.32","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icdm.2015.32","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2015 IEEE International Conference on Data Mining","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1508.07678","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040904107","display_name":"Shahriar Shariat","orcid":"https://orcid.org/0000-0002-5936-1586"},"institutions":[{"id":"https://openalex.org/I4210100604","display_name":"Revolution Medicines (United States)","ror":"https://ror.org/00mny1y94","country_code":"US","type":"company","lineage":["https://openalex.org/I4210100604"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Shahriar Shariat","raw_affiliation_strings":["Turn Inc., Redwood City, CA","Turn Inc., Redwood City, CA, USA"],"affiliations":[{"raw_affiliation_string":"Turn Inc., Redwood City, CA","institution_ids":[]},{"raw_affiliation_string":"Turn Inc., Redwood City, CA, USA","institution_ids":["https://openalex.org/I4210100604"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085127782","display_name":"Burkay Orten","orcid":null},"institutions":[{"id":"https://openalex.org/I4210100604","display_name":"Revolution Medicines (United States)","ror":"https://ror.org/00mny1y94","country_code":"US","type":"company","lineage":["https://openalex.org/I4210100604"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Burkay Orten","raw_affiliation_strings":["Turn Inc., Redwood City, CA","Turn Inc., Redwood City, CA, USA"],"affiliations":[{"raw_affiliation_string":"Turn Inc., Redwood City, CA","institution_ids":[]},{"raw_affiliation_string":"Turn Inc., Redwood City, CA, USA","institution_ids":["https://openalex.org/I4210100604"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043774457","display_name":"Ali Dasdan","orcid":null},"institutions":[{"id":"https://openalex.org/I4210100604","display_name":"Revolution Medicines (United States)","ror":"https://ror.org/00mny1y94","country_code":"US","type":"company","lineage":["https://openalex.org/I4210100604"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ali Dasdan","raw_affiliation_strings":["Turn Inc., Redwood City, CA","Turn Inc., Redwood City, CA, USA"],"affiliations":[{"raw_affiliation_string":"Turn Inc., Redwood City, CA","institution_ids":[]},{"raw_affiliation_string":"Turn Inc., Redwood City, CA, USA","institution_ids":["https://openalex.org/I4210100604"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5040904107"],"corresponding_institution_ids":["https://openalex.org/I4210100604"],"apc_list":null,"apc_paid":null,"fwci":1.1764,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.85691762,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":96},"biblio":{"volume":"2","issue":null,"first_page":"369","last_page":"378"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8144226670265198},{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.7763819694519043},{"id":"https://openalex.org/keywords/bidding","display_name":"Bidding","score":0.7068451642990112},{"id":"https://openalex.org/keywords/purchasing","display_name":"Purchasing","score":0.6889035105705261},{"id":"https://openalex.org/keywords/online-advertising","display_name":"Online advertising","score":0.5906387567520142},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5690363645553589},{"id":"https://openalex.org/keywords/click-through-rate","display_name":"Click-through rate","score":0.5672529339790344},{"id":"https://openalex.org/keywords/display-advertising","display_name":"Display advertising","score":0.5326910018920898},{"id":"https://openalex.org/keywords/return-on-investment","display_name":"Return on investment","score":0.49216312170028687},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.4845227599143982},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.47562721371650696},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.470164030790329},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4504013955593109},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.23412299156188965},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.21800416707992554},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.14361926913261414},{"id":"https://openalex.org/keywords/marketing","display_name":"Marketing","score":0.1345701813697815}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8144226670265198},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.7763819694519043},{"id":"https://openalex.org/C9233905","wikidata":"https://www.wikidata.org/wiki/Q3276328","display_name":"Bidding","level":2,"score":0.7068451642990112},{"id":"https://openalex.org/C2778813691","wikidata":"https://www.wikidata.org/wiki/Q1369832","display_name":"Purchasing","level":2,"score":0.6889035105705261},{"id":"https://openalex.org/C512338625","wikidata":"https://www.wikidata.org/wiki/Q624902","display_name":"Online advertising","level":3,"score":0.5906387567520142},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5690363645553589},{"id":"https://openalex.org/C115174607","wikidata":"https://www.wikidata.org/wiki/Q1100934","display_name":"Click-through rate","level":2,"score":0.5672529339790344},{"id":"https://openalex.org/C2777999536","wikidata":"https://www.wikidata.org/wiki/Q2399498","display_name":"Display advertising","level":4,"score":0.5326910018920898},{"id":"https://openalex.org/C169549615","wikidata":"https://www.wikidata.org/wiki/Q939134","display_name":"Return on investment","level":3,"score":0.49216312170028687},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.4845227599143982},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.47562721371650696},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.470164030790329},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4504013955593109},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.23412299156188965},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21800416707992554},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.14361926913261414},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.1345701813697815},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C2778348673","wikidata":"https://www.wikidata.org/wiki/Q739302","display_name":"Production (economics)","level":2,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C139719470","wikidata":"https://www.wikidata.org/wiki/Q39680","display_name":"Macroeconomics","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/icdm.2015.32","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icdm.2015.32","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2015 IEEE International Conference on Data Mining","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1508.07678","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1508.07678","pdf_url":"https://arxiv.org/pdf/1508.07678","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1508.07678","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1508.07678","pdf_url":"https://arxiv.org/pdf/1508.07678","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.41999998688697815,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W2010953147","https://openalex.org/W2012905273","https://openalex.org/W2039842578","https://openalex.org/W2059527510","https://openalex.org/W2069003154","https://openalex.org/W2105381419","https://openalex.org/W2107328434","https://openalex.org/W2110228583","https://openalex.org/W2112508839","https://openalex.org/W2118020653","https://openalex.org/W2128313432","https://openalex.org/W2135445066","https://openalex.org/W2162979096","https://openalex.org/W2293743194","https://openalex.org/W2315123323","https://openalex.org/W2489417898","https://openalex.org/W2970863555","https://openalex.org/W3124766564","https://openalex.org/W4233555366","https://openalex.org/W4242398740","https://openalex.org/W4320164719","https://openalex.org/W4388295403","https://openalex.org/W6683646410","https://openalex.org/W6766905983"],"related_works":["https://openalex.org/W2100597815","https://openalex.org/W175164097","https://openalex.org/W2143648166","https://openalex.org/W2952316437","https://openalex.org/W4300589523","https://openalex.org/W2591602503","https://openalex.org/W2162862818","https://openalex.org/W3120879483","https://openalex.org/W2078847107","https://openalex.org/W2096414357"],"abstract_inverted_index":{"Online":[0],"media":[1],"provides":[2],"opportunities":[3],"for":[4,50,108,116,131,180,234],"marketers":[5],"through":[6,282],"which":[7],"they":[8],"can":[9],"deliver":[10],"effective":[11],"brand":[12],"messages":[13],"to":[14,27,36,44,54,158,261],"a":[15,51,159,191,230,267],"wide":[16],"range":[17],"of":[18,86,101,143,161,237,250,274],"audiences":[19],"at":[20,57],"scale.":[21],"Advertising":[22],"technology":[23],"platforms":[24],"enable":[25],"advertisers":[26],"reach":[28],"their":[29],"target":[30],"audience":[31],"by":[32],"delivering":[33],"ad":[34],"impressions":[35,56],"online":[37,103,235],"users":[38],"in":[39,79,95,154,226,266],"real":[40,279],"time.":[41],"In":[42,123],"order":[43],"identify":[45],"the":[46,58,72,80,82,118,140,149,177,242,272],"best":[47],"marketing":[48,119],"message":[49],"user":[52],"and":[53,67,98,176,198,222,228,253],"purchase":[55],"right":[59],"price,":[60],"we":[61,126,147,152,184,217],"rely":[62],"heavily":[63],"on":[64,139,187,278],"bid":[65,73,132],"prediction":[66,74,133],"optimization":[68],"models.":[69],"Even":[70],"though":[71],"models":[75,134,240],"are":[76],"well":[77,114],"studied":[78],"literature,":[81],"equally":[83],"important":[84],"subject":[85],"model":[87,105,111,144,193],"evaluation":[88,100,221,236,276],"is":[89,106,138],"usually":[90],"overlooked":[91],"or":[92,211],"not":[93],"discussed":[94],"detail.":[96],"Effective":[97],"reliable":[99,268],"an":[102,128],"bidding":[104],"crucial":[107],"making":[109],"faster":[110],"improvements":[112],"as":[113,115,164,190,207],"utilizing":[117],"budgets":[120],"more":[121,203,231,264],"efficiently.":[122],"this":[124],"paper,":[125],"present":[127],"experimentation":[129],"framework":[130],"where":[135],"our":[136,155,275],"focus":[137],"practical":[141],"aspects":[142],"evaluation.":[145],"Specifically,":[146],"outline":[148],"unique":[150],"challenges":[151],"encounter":[153],"platform":[156],"due":[157],"variety":[160],"factors":[162],"such":[163,206],"heterogeneous":[165],"goal":[166],"definitions,":[167],"varying":[168],"budget":[169],"requirements":[170],"across":[171],"different":[172],"campaigns,":[173],"high":[174],"seasonality":[175],"auction-based":[178],"environment":[179],"inventory":[181],"purchasing.":[182],"Then,":[183],"introduce":[185],"return":[186],"investment":[188],"(ROI)":[189],"unified":[192],"performance":[194],"(i.e.,":[195],"success)":[196],"metric":[197,223],"explain":[199],"its":[200],"merits":[201],"over":[202],"traditional":[204],"metrics":[205],"click-through":[208],"rate":[209,213],"(CTR)":[210],"conversion":[212],"(CVR).":[214],"Most":[215],"importantly,":[216],"discuss":[218],"commonly":[219],"used":[220],"summarization":[224],"approaches":[225],"detail":[227],"propose":[229],"accurate":[232],"method":[233],"new":[238],"experimental":[239],"against":[241],"baseline.":[243],"Our":[244],"meta-analysis-based":[245],"approach":[246],"addresses":[247],"various":[248],"shortcomings":[249],"other":[251],"methods":[252],"yields":[254],"statistically":[255],"robust":[256],"conclusions":[257],"that":[258],"allow":[259],"us":[260],"conclude":[262],"experiments":[263],"quickly":[265],"manner.":[269],"We":[270],"demonstrate":[271],"effectiveness":[273],"strategy":[277],"campaign":[280],"data":[281],"some":[283],"experiments.":[284]},"counts_by_year":[{"year":2016,"cited_by_count":2}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}