{"id":"https://openalex.org/W3012688973","doi":"https://doi.org/10.1145/3366423.3380125","title":"Adversarial Bandits Policy for Crawling Commercial Web Content","display_name":"Adversarial Bandits Policy for Crawling Commercial Web Content","publication_year":2020,"publication_date":"2020-04-20","ids":{"openalex":"https://openalex.org/W3012688973","doi":"https://doi.org/10.1145/3366423.3380125","mag":"3012688973"},"language":"en","primary_location":{"id":"doi:10.1145/3366423.3380125","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366423.3380125","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Web Conference 2020","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3366423.3380125","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071179467","display_name":"Shuguang Han","orcid":"https://orcid.org/0000-0003-1416-6960"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Shuguang Han","raw_affiliation_strings":["Google, USA"],"affiliations":[{"raw_affiliation_string":"Google, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032248436","display_name":"Michael Bendersky","orcid":"https://orcid.org/0000-0002-2941-6240"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Bendersky","raw_affiliation_strings":["Google, USA"],"affiliations":[{"raw_affiliation_string":"Google, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070532256","display_name":"Przemek Gajda","orcid":null},"institutions":[{"id":"https://openalex.org/I4210100430","display_name":"Google (Switzerland)","ror":"https://ror.org/014f9c269","country_code":"CH","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210100430","https://openalex.org/I4210128969"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Przemek Gajda","raw_affiliation_strings":["Google, Switzerland"],"affiliations":[{"raw_affiliation_string":"Google, Switzerland","institution_ids":["https://openalex.org/I4210100430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028327226","display_name":"S. P. Novikov","orcid":"https://orcid.org/0000-0002-3981-7873"},"institutions":[{"id":"https://openalex.org/I4210100430","display_name":"Google (Switzerland)","ror":"https://ror.org/014f9c269","country_code":"CH","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210100430","https://openalex.org/I4210128969"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Sergey Novikov","raw_affiliation_strings":["Google, Switzerland"],"affiliations":[{"raw_affiliation_string":"Google, Switzerland","institution_ids":["https://openalex.org/I4210100430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037200145","display_name":"Marc Najork","orcid":"https://orcid.org/0000-0003-1423-0854"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Marc Najork","raw_affiliation_strings":["Google, USA"],"affiliations":[{"raw_affiliation_string":"Google, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042628219","display_name":"Bernhard Brodowsky","orcid":null},"institutions":[{"id":"https://openalex.org/I4210100430","display_name":"Google (Switzerland)","ror":"https://ror.org/014f9c269","country_code":"CH","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210100430","https://openalex.org/I4210128969"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Bernhard Brodowsky","raw_affiliation_strings":["Google, Switzerland"],"affiliations":[{"raw_affiliation_string":"Google, Switzerland","institution_ids":["https://openalex.org/I4210100430"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027411355","display_name":"Alexandrin Popescul","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alexandrin Popescul","raw_affiliation_strings":["Pinterest, USA"],"affiliations":[{"raw_affiliation_string":"Pinterest, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5071179467"],"corresponding_institution_ids":["https://openalex.org/I1291425158"],"apc_list":null,"apc_paid":null,"fwci":0.3714,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.6451068,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"407","last_page":"417"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12288","display_name":"Optimization and Search Problems","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9865000247955322,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8130519390106201},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.7093684077262878},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5755104422569275},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.5248727202415466},{"id":"https://openalex.org/keywords/web-content","display_name":"Web content","score":0.5169517397880554},{"id":"https://openalex.org/keywords/crawling","display_name":"Crawling","score":0.4898805022239685},{"id":"https://openalex.org/keywords/resource","display_name":"Resource (disambiguation)","score":0.4849167466163635},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.4480663537979126},{"id":"https://openalex.org/keywords/product","display_name":"Product (mathematics)","score":0.437260240316391},{"id":"https://openalex.org/keywords/service","display_name":"Service (business)","score":0.4126926362514496},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2684285640716553},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.22425362467765808},{"id":"https://openalex.org/keywords/web-page","display_name":"Web page","score":0.20228832960128784},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.12471193075180054},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.0907449722290039}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8130519390106201},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.7093684077262878},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5755104422569275},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.5248727202415466},{"id":"https://openalex.org/C2776324614","wikidata":"https://www.wikidata.org/wiki/Q3948731","display_name":"Web content","level":3,"score":0.5169517397880554},{"id":"https://openalex.org/C100368936","wikidata":"https://www.wikidata.org/wiki/Q1411725","display_name":"Crawling","level":2,"score":0.4898805022239685},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.4849167466163635},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.4480663537979126},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.437260240316391},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.4126926362514496},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2684285640716553},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.22425362467765808},{"id":"https://openalex.org/C21959979","wikidata":"https://www.wikidata.org/wiki/Q36774","display_name":"Web page","level":2,"score":0.20228832960128784},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.12471193075180054},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0907449722290039},{"id":"https://openalex.org/C105702510","wikidata":"https://www.wikidata.org/wiki/Q514","display_name":"Anatomy","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3366423.3380125","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366423.3380125","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Web Conference 2020","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3366423.3380125","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366423.3380125","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Web Conference 2020","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W13754670","https://openalex.org/W757809383","https://openalex.org/W1484417080","https://openalex.org/W1485536830","https://openalex.org/W1566984846","https://openalex.org/W1608864735","https://openalex.org/W1660264423","https://openalex.org/W1669104078","https://openalex.org/W1938006511","https://openalex.org/W1970841368","https://openalex.org/W1976624301","https://openalex.org/W1987272746","https://openalex.org/W1988497390","https://openalex.org/W2005796248","https://openalex.org/W2016122268","https://openalex.org/W2018928332","https://openalex.org/W2029341294","https://openalex.org/W2030453570","https://openalex.org/W2038378248","https://openalex.org/W2040716668","https://openalex.org/W2049934117","https://openalex.org/W2059713800","https://openalex.org/W2074694452","https://openalex.org/W2077902449","https://openalex.org/W2080901490","https://openalex.org/W2112420033","https://openalex.org/W2118129497","https://openalex.org/W2119058983","https://openalex.org/W2127536142","https://openalex.org/W2140279085","https://openalex.org/W2142211218","https://openalex.org/W2148931201","https://openalex.org/W2165862792","https://openalex.org/W2169557227","https://openalex.org/W2402144811","https://openalex.org/W2473259777","https://openalex.org/W2591610401","https://openalex.org/W2604851540","https://openalex.org/W2740070748","https://openalex.org/W2884612586","https://openalex.org/W2911388033","https://openalex.org/W2911517843","https://openalex.org/W2947460167","https://openalex.org/W2948596272","https://openalex.org/W2949213912","https://openalex.org/W2950577311","https://openalex.org/W2950929549","https://openalex.org/W2951371685","https://openalex.org/W4213362702","https://openalex.org/W4242841123","https://openalex.org/W4251434705"],"related_works":["https://openalex.org/W4393220254","https://openalex.org/W4321258516","https://openalex.org/W2051833850","https://openalex.org/W2502115930","https://openalex.org/W4287845917","https://openalex.org/W3156164993","https://openalex.org/W2385015894","https://openalex.org/W2171573941","https://openalex.org/W2482350142","https://openalex.org/W2608427695"],"abstract_inverted_index":{"The":[0],"rapid":[1],"growth":[2],"of":[3,11,26,54,99,132],"commercial":[4,27],"web":[5],"content":[6,84,110,133],"has":[7],"driven":[8],"the":[9,23,49,55,65,97,118,136,145,179,188],"development":[10],"shopping":[12,39],"search":[13,40],"services":[14],"to":[15,22,48,82,205],"help":[16],"users":[17,45],"find":[18],"product":[19,51],"offers.":[20],"Due":[21],"dynamic":[24],"nature":[25],"content,":[28],"an":[29,77],"effective":[30],"recrawl":[31,160,172],"policy":[32],"is":[33,101,154],"a":[34,38,87,129,155,169,198],"key":[35],"component":[36],"in":[37,103],"service;":[41],"it":[42,162],"ensures":[43],"that":[44,96,151,184],"have":[46],"access":[47],"up-to-date":[50],"details.":[52],"Most":[53],"existing":[56,159],"strategies":[57],"either":[58],"relied":[59],"on":[60],"simple":[61],"heuristics,":[62],"or":[63],"overlooked":[64],"resource":[66,89,210],"budgets.":[67,211],"To":[68,174],"address":[69],"this,":[70],"Azar":[71],"et":[72],"al.":[73],"[5]":[74],"recently":[75],"proposed":[76],"optimization":[78],"strategy":[79],"LambdaCrawl":[80,100,138,153],"aiming":[81],"maximize":[83],"freshness":[85,134,190],"within":[86],"given":[88],"budget.":[90],"In":[91],"this":[92,175],"paper,":[93],"we":[94,127,149,177],"demonstrate":[95,150],"effectiveness":[98],"governed":[102],"large":[104],"part":[105],"by":[106,168,191],"how":[107],"well":[108],"future":[109],"change":[111,124,141],"rate":[112,125,142],"can":[113,163,185],"be":[114,164],"estimated.":[115],"By":[116],"adopting":[117],"state-of-the-art":[119],"deep":[120],"learning":[121],"models":[122],"for":[123],"prediction,":[126],"obtain":[128],"substantial":[130],"increase":[131],"over":[135,197],"common":[137],"implementation":[139],"with":[140],"estimated":[143],"from":[144],"past":[146],"history.":[147],"Moreover,":[148],"while":[152],"significant":[156],"advancement":[157],"upon":[158,167],"strategies,":[161],"further":[165],"improved":[166],"unified":[170],"multi-strategy":[171],"policy.":[173],"end,":[176],"adopt":[178],"K-armed":[180],"adversarial":[181],"bandits":[182],"algorithm":[183],"provably":[186],"optimize":[187],"overall":[189],"combining":[192],"multiple":[193],"strategies.":[194],"Empirical":[195],"results":[196],"large-scale":[199],"production":[200],"dataset":[201],"confirm":[202],"its":[203],"superiority":[204],"LambdaCrawl,":[206],"especially":[207],"under":[208],"tight":[209]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}