{"id":"https://openalex.org/W2952252579","doi":"https://doi.org/10.48550/arxiv.1307.6080","title":"Timely crawling of high-quality ephemeral new content","display_name":"Timely crawling of high-quality ephemeral new content","publication_year":2013,"publication_date":"2013-07-23","ids":{"openalex":"https://openalex.org/W2952252579","doi":"https://doi.org/10.48550/arxiv.1307.6080","mag":"2952252579"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:1307.6080","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1307.6080","pdf_url":"https://arxiv.org/pdf/1307.6080","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1307.6080","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045668591","display_name":"Damien Lefortier","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I58957048","display_name":"Yandex (Russia)","ror":"https://ror.org/04dbch786","country_code":"RU","type":"company","lineage":["https://openalex.org/I58957048"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Lefortier, Damien","raw_affiliation_strings":["Yandex, Moscow, Russian Fed"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yandex, Moscow, Russian Fed","institution_ids":["https://openalex.org/I58957048"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ostroumova, Liudmila","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I19880235","display_name":"Lomonosov Moscow State University","ror":"https://ror.org/010pmpe69","country_code":"RU","type":"education","lineage":["https://openalex.org/I19880235"]},{"id":"https://openalex.org/I58957048","display_name":"Yandex (Russia)","ror":"https://ror.org/04dbch786","country_code":"RU","type":"company","lineage":["https://openalex.org/I58957048"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Ostroumova, Liudmila","raw_affiliation_strings":["Moscow State University","Yandex","Yandex, Moscow, Russian Fed"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Moscow State University","institution_ids":["https://openalex.org/I19880235"]},{"raw_affiliation_string":"Yandex","institution_ids":["https://openalex.org/I58957048"]},{"raw_affiliation_string":"Yandex, Moscow, Russian Fed","institution_ids":["https://openalex.org/I58957048"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022976447","display_name":"Egor Samosvat","orcid":"https://orcid.org/0000-0003-3399-4235","observed_orcids":["https://orcid.org/0000-0003-3399-4235"]},"institutions":[{"id":"https://openalex.org/I153845743","display_name":"Moscow Institute of Physics and Technology","ror":"https://ror.org/00v0z9322","country_code":"RU","type":"education","lineage":["https://openalex.org/I153845743"]},{"id":"https://openalex.org/I58957048","display_name":"Yandex (Russia)","ror":"https://ror.org/04dbch786","country_code":"RU","type":"company","lineage":["https://openalex.org/I58957048"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Samosvat, Egor","raw_affiliation_strings":["Moscow Institute of Physics and Technology","Yandex","Yandex, Moscow, Russian Fed"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Moscow Institute of Physics and Technology","institution_ids":["https://openalex.org/I153845743"]},{"raw_affiliation_string":"Yandex","institution_ids":["https://openalex.org/I58957048"]},{"raw_affiliation_string":"Yandex, Moscow, Russian Fed","institution_ids":["https://openalex.org/I58957048"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069551961","display_name":"Pavel Serdyukov","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I58957048","display_name":"Yandex (Russia)","ror":"https://ror.org/04dbch786","country_code":"RU","type":"company","lineage":["https://openalex.org/I58957048"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Serdyukov, Pavel","raw_affiliation_strings":["Yandex, Moscow, Russian Fed"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yandex, Moscow, Russian Fed","institution_ids":["https://openalex.org/I58957048"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.0010999999940395355,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13976","display_name":"Web visibility and informetrics","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/web-crawling","display_name":"web crawling","score":0.8889999985694885},{"id":"https://openalex.org/keywords/ephemeral-content","display_name":"ephemeral content","score":0.8519999980926514},{"id":"https://openalex.org/keywords/resource-allocation","display_name":"resource allocation","score":0.7059999704360962},{"id":"https://openalex.org/keywords/click-logs","display_name":"click logs","score":0.8479999899864197},{"id":"https://openalex.org/keywords/search-engine-indexing","display_name":"search engine indexing","score":0.75}],"concepts":[{"id":"https://openalex.org/C100368936","wikidata":"https://www.wikidata.org/wiki/Q1411725","display_name":"Crawling","level":2,"score":0.8927513360977173},{"id":"https://openalex.org/C13743948","wikidata":"https://www.wikidata.org/wiki/Q45842","display_name":"Web crawler","level":2,"score":0.7409019470214844},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7231006026268005},{"id":"https://openalex.org/C76947770","wikidata":"https://www.wikidata.org/wiki/Q4533181","display_name":"Ephemeral key","level":2,"score":0.6568065881729126},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.640436053276062},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5579318404197693},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.553275465965271},{"id":"https://openalex.org/C21959979","wikidata":"https://www.wikidata.org/wiki/Q36774","display_name":"Web page","level":2,"score":0.5158142447471619},{"id":"https://openalex.org/C75165309","wikidata":"https://www.wikidata.org/wiki/Q2258979","display_name":"Search engine indexing","level":2,"score":0.4742647409439087},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.46157678961753845},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.44571706652641296},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.43899938464164734},{"id":"https://openalex.org/C73340581","wikidata":"https://www.wikidata.org/wiki/Q5463958","display_name":"Focused crawler","level":5,"score":0.4302181899547577},{"id":"https://openalex.org/C2776324614","wikidata":"https://www.wikidata.org/wiki/Q3948731","display_name":"Web content","level":3,"score":0.4158775508403778},{"id":"https://openalex.org/C61096286","wikidata":"https://www.wikidata.org/wiki/Q7978592","display_name":"Web navigation","level":3,"score":0.23294347524642944},{"id":"https://openalex.org/C173576120","wikidata":"https://www.wikidata.org/wiki/Q2641220","display_name":"Static web page","level":4,"score":0.21900519728660583},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.13080856204032898},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.08231306076049805},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C105702510","wikidata":"https://www.wikidata.org/wiki/Q514","display_name":"Anatomy","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:1307.6080","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1307.6080","pdf_url":"https://arxiv.org/pdf/1307.6080","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2952252579","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/1307.6080","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1307.6080","doi":"https://doi.org/10.48550/arxiv.1307.6080","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1307.6080","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1307.6080","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1307.6080","pdf_url":"https://arxiv.org/pdf/1307.6080","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2952252579.pdf","grobid_xml":"https://content.openalex.org/works/W2952252579.grobid-xml"},"referenced_works_count":16,"referenced_works":["https://openalex.org/W14315322","https://openalex.org/W14978841","https://openalex.org/W1970841368","https://openalex.org/W1976624301","https://openalex.org/W1987272746","https://openalex.org/W2059713800","https://openalex.org/W2099126271","https://openalex.org/W2102121847","https://openalex.org/W2107170340","https://openalex.org/W2126075801","https://openalex.org/W2127536142","https://openalex.org/W2128384372","https://openalex.org/W2151932833","https://openalex.org/W2155467656","https://openalex.org/W2158601853","https://openalex.org/W2159869159"],"related_works":["https://openalex.org/W7100431525","https://openalex.org/W2127536142","https://openalex.org/W1702782474","https://openalex.org/W2030453570","https://openalex.org/W7099267392","https://openalex.org/W2394675604","https://openalex.org/W2949213912","https://openalex.org/W4367310752","https://openalex.org/W2119012894","https://openalex.org/W2121162176"],"abstract_inverted_index":{"Nowadays,":[0],"more":[1,3],"and":[2,20,48,74,98,106,156,173,215],"people":[4],"use":[5],"the":[6,69,128,192,210,219,236],"Web":[7,25],"as":[8],"their":[9],"primary":[10],"source":[11],"of":[12,22,41,71,76,83,130,153,186,191,241],"up-to-date":[13],"information.":[14],"In":[15,64],"this":[16,65,122],"context,":[17],"fast":[18],"crawling":[19,75,87],"indexing":[21],"newly":[23,175],"created":[24,176],"pages":[26,44,80,97,135,144,177],"has":[27],"become":[28],"crucial":[29],"for":[30,61,121,133,160],"search":[31,225],"engines,":[32],"especially":[33],"because":[34],"user":[35,84,131,187,238],"traffic":[36],"to":[37,95,168,197,229],"a":[38,115,149,158,163],"significant":[39],"fraction":[40],"these":[42,201],"new":[43,79,116,143],"(like":[45],"news,":[46],"blog":[47],"forum":[49],"posts)":[50],"grows":[51],"really":[52],"quickly":[53,104],"right":[54],"after":[55],"they":[56],"appear,":[57],"but":[58],"lasts":[59],"only":[60],"several":[62],"days.":[63],"paper,":[66],"we":[67,223],"study":[68],"problem":[70],"timely":[72],"finding":[73,161],"such":[77,96,162],"ephemeral":[78,134,142],"(in":[81,184],"terms":[82,185],"interest).":[85],"Traditional":[86],"policies":[88],"do":[89],"not":[90,103],"give":[91,230],"any":[92],"particular":[93],"priority":[94],"may":[99],"thus":[100,113],"crawl":[101,108,174],"them":[102],"enough,":[105],"even":[107],"already":[109],"obsolete":[110],"content.":[111,189],"We":[112,138,208],"propose":[114],"metric,":[117],"well":[118],"thought":[119],"out":[120],"task,":[123],"which":[124],"takes":[125],"into":[126],"account":[127],"decrease":[129],"interest":[132],"over":[136],"time.":[137],"show":[139],"that":[140],"most":[141],"can":[145],"be":[146],"found":[147],"at":[148],"relatively":[150],"small":[151],"set":[152],"content":[154,171],"sources":[155,172],"present":[157],"procedure":[159],"set.":[164],"Our":[165],"idea":[166],"is":[167,196,244],"periodically":[169],"recrawl":[170],"linked":[178],"from":[179],"them,":[180],"focusing":[181],"on":[182,248],"high-quality":[183],"interest)":[188],"One":[190],"main":[193],"difficulties":[194],"here":[195],"divide":[198],"resources":[199],"between":[200,213],"two":[202],"activities":[203],"in":[204],"an":[205,233],"efficient":[206],"way.":[207],"find":[209],"adaptive":[211],"balance":[212],"crawls":[214],"recrawls":[216],"by":[217],"maximizing":[218],"proposed":[220],"metric.":[221],"Further,":[222],"incorporate":[224],"engine":[226],"click":[227],"logs":[228],"our":[231,242],"crawler":[232],"insight":[234],"about":[235],"current":[237],"demands.":[239],"Efficiency":[240],"approach":[243],"finally":[245],"demonstrated":[246],"experimentally":[247],"real-world":[249],"data.":[250]},"counts_by_year":[],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2019-06-27T00:00:00"}