{"id":"https://openalex.org/W2098293053","doi":"https://doi.org/10.1145/1458502.1458511","title":"High-performance priority queues for parallel crawlers","display_name":"High-performance priority queues for parallel crawlers","publication_year":2008,"publication_date":"2008-10-30","ids":{"openalex":"https://openalex.org/W2098293053","doi":"https://doi.org/10.1145/1458502.1458511","mag":"2098293053"},"language":"en","primary_location":{"id":"doi:10.1145/1458502.1458511","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1458502.1458511","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 10th ACM workshop on Web information and data management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044981704","display_name":"Mauricio Mar\u0131\u0301n","orcid":"https://orcid.org/0000-0003-0662-7149"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mauricio Marin","raw_affiliation_strings":["Yahoo! Research Latin America, Santiago, Chile","Yahoo! Research Latin-America, Santiago, Chile"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yahoo! Research Latin America, Santiago, Chile","institution_ids":[]},{"raw_affiliation_string":"Yahoo! Research Latin-America, Santiago, Chile","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081756481","display_name":"Rodr\u00edgo Paredes","orcid":"https://orcid.org/0000-0002-9943-2510"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rodrigo Paredes","raw_affiliation_strings":["Yahoo! Research Latin America, Santiago, Chile","Yahoo! Research Latin-America, Santiago, Chile"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yahoo! Research Latin America, Santiago, Chile","institution_ids":[]},{"raw_affiliation_string":"Yahoo! Research Latin-America, Santiago, Chile","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5104440477","display_name":"Carolina Bonacic","orcid":null},"institutions":[{"id":"https://openalex.org/I121748325","display_name":"Universidad Complutense de Madrid","ror":"https://ror.org/02p0gd045","country_code":"ES","type":"education","lineage":["https://openalex.org/I121748325"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Carolina Bonacic","raw_affiliation_strings":["Complutense University of Madrid, Madrid, Spain","Complutense University of Madrid. Madrid. Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Complutense University of Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I121748325"]},{"raw_affiliation_string":"Complutense University of Madrid. Madrid. Spain","institution_ids":["https://openalex.org/I121748325"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.8813,"has_fulltext":false,"cited_by_count":19,"citation_normalized_percentile":{"value":0.97037409,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"47","last_page":"54"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8761375546455383},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6858347654342651},{"id":"https://openalex.org/keywords/download","display_name":"Download","score":0.5812209248542786},{"id":"https://openalex.org/keywords/node","display_name":"Node (physics)","score":0.5491828322410583},{"id":"https://openalex.org/keywords/priority-queue","display_name":"Priority queue","score":0.5403715372085571},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.4766883850097656},{"id":"https://openalex.org/keywords/queue","display_name":"Queue","score":0.4504035413265228},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.4363037347793579},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.4266579747200012},{"id":"https://openalex.org/keywords/server","display_name":"Server","score":0.4261142611503601},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.34881582856178284}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8761375546455383},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6858347654342651},{"id":"https://openalex.org/C2780154274","wikidata":"https://www.wikidata.org/wiki/Q7126717","display_name":"Download","level":2,"score":0.5812209248542786},{"id":"https://openalex.org/C62611344","wikidata":"https://www.wikidata.org/wiki/Q1062658","display_name":"Node (physics)","level":2,"score":0.5491828322410583},{"id":"https://openalex.org/C26951474","wikidata":"https://www.wikidata.org/wiki/Q629283","display_name":"Priority queue","level":3,"score":0.5403715372085571},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.4766883850097656},{"id":"https://openalex.org/C160403385","wikidata":"https://www.wikidata.org/wiki/Q220543","display_name":"Queue","level":2,"score":0.4504035413265228},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4363037347793579},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.4266579747200012},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.4261142611503601},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.34881582856178284},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.0},{"id":"https://openalex.org/C66938386","wikidata":"https://www.wikidata.org/wiki/Q633538","display_name":"Structural engineering","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/1458502.1458511","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1458502.1458511","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 10th ACM workshop on Web information and data management","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.140.5286","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.140.5286","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.dcc.uchile.cl/~mmarin/papers/widm2008.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W13754670","https://openalex.org/W1542986586","https://openalex.org/W1595621137","https://openalex.org/W1792966054","https://openalex.org/W2000273502","https://openalex.org/W2016589434","https://openalex.org/W2045271686","https://openalex.org/W2066636486","https://openalex.org/W2100848813","https://openalex.org/W2113184419","https://openalex.org/W2152582590","https://openalex.org/W2158601853","https://openalex.org/W2295141584","https://openalex.org/W2407942522","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W2092530219","https://openalex.org/W2388464034","https://openalex.org/W2533125852","https://openalex.org/W2140460949","https://openalex.org/W2105580438","https://openalex.org/W1980801634","https://openalex.org/W189328514","https://openalex.org/W2143289692","https://openalex.org/W3049636605","https://openalex.org/W1974858599"],"abstract_inverted_index":{"Large":[0],"scale":[1],"data":[2],"centers":[3],"for":[4],"crawlers":[5],"are":[6,112],"able":[7],"to":[8,20,49,75,84],"maintain":[9],"a":[10,41,79,108],"very":[11],"large":[12],"number":[13,29],"of":[14,30,36,44,47,110,115,138],"active":[15],"http":[16],"connections":[17],"in":[18,82],"order":[19,83],"download":[21,85],"as":[22,24,100],"fast":[23],"possible":[25],"the":[26,37,57,86,113],"usually":[27],"huge":[28],"web":[31],"pages":[32],"from":[33],"given":[34],"sections":[35],"WWW.":[38],"This":[39],"generates":[40],"continuous":[42],"stream":[43],"new":[45,72],"URLs":[46,73,101],"documents":[48,89],"be":[50,62,76],"downloaded":[51,102],"and":[52,121],"it":[53],"is":[54],"clear":[55],"that":[56],"associated":[58],"work-load":[59],"can":[60],"only":[61],"served":[63],"efficiently":[64],"with":[65,95],"proper":[66],"parallel":[67],"computing":[68],"techniques.":[69],"The":[70],"incoming":[71],"have":[74],"organized":[77],"by":[78,103],"priority":[80],"measure":[81],"most":[87],"relevant":[88],"first.":[90],"Efficiently":[91],"managing":[92],"them":[93],"along":[94],"other":[96],"synchronization":[97],"issues":[98],"such":[99],"different":[104],"processing":[105],"nodes":[106],"forming":[107],"cluster":[109],"computers":[111],"matters":[114],"this":[116],"paper.":[117],"We":[118],"propose":[119],"efficient":[120,136],"scalable":[122],"strategies":[123],"which":[124],"consider":[125],"intra-node":[126],"multi-core":[127],"multi-threading":[128],"on":[129],"an":[130],"inter-nodes":[131],"distributed":[132],"memory":[133],"environment,":[134],"including":[135],"use":[137],"secondary":[139],"memory.":[140]},"counts_by_year":[{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":2},{"year":2014,"cited_by_count":2},{"year":2013,"cited_by_count":2},{"year":2012,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}