{"id":"https://openalex.org/W2049574141","doi":"https://doi.org/10.1145/1741906.1741937","title":"A scalable, extensible web crawler based on P2P overlay networks","display_name":"A scalable, extensible web crawler based on P2P overlay networks","publication_year":2010,"publication_date":"2010-02-26","ids":{"openalex":"https://openalex.org/W2049574141","doi":"https://doi.org/10.1145/1741906.1741937","mag":"2049574141"},"language":"en","primary_location":{"id":"doi:10.1145/1741906.1741937","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1741906.1741937","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Conference and Workshop on Emerging Trends in Technology","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015619835","display_name":"Prateek Mittal","orcid":"https://orcid.org/0000-0002-4057-0118"},"institutions":[{"id":"https://openalex.org/I55124831","display_name":"J.C. Bose University of Science & Technology, YMCA","ror":"https://ror.org/014jqnm52","country_code":"IN","type":"education","lineage":["https://openalex.org/I55124831"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"P. Mittal","raw_affiliation_strings":["YMCAIE, Faridabad, Haryana, India"],"affiliations":[{"raw_affiliation_string":"YMCAIE, Faridabad, Haryana, India","institution_ids":["https://openalex.org/I55124831"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085450587","display_name":"Ashutosh Dixit","orcid":"https://orcid.org/0000-0001-8873-4348"},"institutions":[{"id":"https://openalex.org/I55124831","display_name":"J.C. Bose University of Science & Technology, YMCA","ror":"https://ror.org/014jqnm52","country_code":"IN","type":"education","lineage":["https://openalex.org/I55124831"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"A. Dixit","raw_affiliation_strings":["YMCAIE, Fbd., Haryana, India"],"affiliations":[{"raw_affiliation_string":"YMCAIE, Fbd., Haryana, India","institution_ids":["https://openalex.org/I55124831"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5114086312","display_name":"A. K. Sharma","orcid":"https://orcid.org/0000-0001-5206-4645"},"institutions":[{"id":"https://openalex.org/I55124831","display_name":"J.C. Bose University of Science & Technology, YMCA","ror":"https://ror.org/014jqnm52","country_code":"IN","type":"education","lineage":["https://openalex.org/I55124831"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"A. K. Sharma","raw_affiliation_strings":["YMCAIE, Fbd., Haryana, India"],"affiliations":[{"raw_affiliation_string":"YMCAIE, Fbd., Haryana, India","institution_ids":["https://openalex.org/I55124831"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5015619835"],"corresponding_institution_ids":["https://openalex.org/I55124831"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.14638174,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"159","last_page":"162"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10742","display_name":"Peer-to-Peer Network Technologies","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8276423811912537},{"id":"https://openalex.org/keywords/mercator-projection","display_name":"Mercator projection","score":0.7855362892150879},{"id":"https://openalex.org/keywords/web-crawler","display_name":"Web crawler","score":0.6021177768707275},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.583406388759613},{"id":"https://openalex.org/keywords/overlay-network","display_name":"Overlay network","score":0.47415488958358765},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.46703478693962097},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.45619305968284607},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.3228442966938019},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.31903478503227234}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8276423811912537},{"id":"https://openalex.org/C36639349","wikidata":"https://www.wikidata.org/wiki/Q309372","display_name":"Mercator projection","level":2,"score":0.7855362892150879},{"id":"https://openalex.org/C13743948","wikidata":"https://www.wikidata.org/wiki/Q45842","display_name":"Web crawler","level":2,"score":0.6021177768707275},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.583406388759613},{"id":"https://openalex.org/C169851745","wikidata":"https://www.wikidata.org/wiki/Q1331985","display_name":"Overlay network","level":3,"score":0.47415488958358765},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.46703478693962097},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.45619305968284607},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.3228442966938019},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.31903478503227234},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/1741906.1741937","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1741906.1741937","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Conference and Workshop on Emerging Trends in Technology","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":7,"referenced_works":["https://openalex.org/W128720162","https://openalex.org/W162038026","https://openalex.org/W1613836731","https://openalex.org/W2066636486","https://openalex.org/W2079333567","https://openalex.org/W2164542999","https://openalex.org/W3165489621"],"related_works":["https://openalex.org/W2353981351","https://openalex.org/W2601929278","https://openalex.org/W2481961680","https://openalex.org/W1586105736","https://openalex.org/W2008777395","https://openalex.org/W4285726220","https://openalex.org/W3210526140","https://openalex.org/W2074095185","https://openalex.org/W3035635662","https://openalex.org/W1956155274"],"abstract_inverted_index":{"The":[0,26,281],"World":[1],"Wide":[2],"Web":[3,42],"is":[4,51,87,124,162,180,242,247,253],"an":[5,21,178,240],"interlinked":[6],"collection":[7,18],"of":[8,10,16,33,47,114,196,232,259,295],"billions":[9],"documents.":[11],"Ironically":[12],"the":[13,38,45,66,112,151,163,188,194,230,257,270,293,296,301],"very":[14],"size":[15],"this":[17,176,238,250],"has":[19,28,98],"become":[20],"obstacle":[22],"for":[23,59,168,203,212,268],"information":[24,39],"retrieval.":[25],"user":[27],"to":[29,35,131,154,159,186,300],"sift":[30],"through":[31,244,276],"scores":[32],"pages":[34],"come":[36],"upon":[37],"he/she":[40],"desires.":[41],"crawlers":[43],"are":[44,226],"heart":[46],"search":[48],"engines.":[49],"Mercator":[50,77,97],"a":[52,73,92,125,169,198,209,220,265,289],"scalable,":[53],"extensible":[54],"web":[55],"crawler,":[56],"which":[57,138,161,182,225,245],"support":[58],"extensibility":[60,275],"and":[61,68,142,207,274,278],"customizability.":[62],"This":[63,108,190],"paper":[64,109],"explores":[65,111,193],"challenges":[67],"issues":[69,102],"faced":[70],"in":[71,76,83,116,127,292],"using":[72,105,175,197,229,237,256],"single":[74,106,121,199],"FIFO":[75,200],"as":[78,91,298],"URL":[79,85,94,117,122,148,211,223],"frontier.":[80],"In":[81],"addition,":[82],"Mercator,":[84],"frontier":[86,118,123,149],"traversed":[88],"every":[89],"time":[90],"new":[93],"arrives.":[95],"Since":[96,120],"multithreaded":[99,128],"environment,":[100],"many":[101],"arises":[103],"by":[104,174,228,236,255],"FIFO.":[107],"also":[110,192],"concept":[113,152,231,258],"pastry":[115],"implementation.":[119],"constraint":[126],"environment.":[129],"Peer":[130],"peer":[132,153,155],"overlay":[133,156],"networks":[134],"provide":[135],"locality":[136],"property,":[137],"improves":[139],"application":[140],"performance":[141,294],"reduces":[143],"network":[144],"usage.":[145],"Hence":[146,173,235],"Mercator's":[147],"uses":[150,183],"network's":[157],"logic":[158,177,239],"find":[160,187],"best":[164],"suitable":[165],"canonical":[166,210,222],"form":[167],"newly":[170],"arrived":[171],"URL.":[172],"algorithm":[179,241],"designed":[181,243],"minimum":[184],"comparisons":[185],"match.":[189],"work":[191],"idea":[195],"sub":[201],"queue":[202],"each":[204,213],"working":[205,214,217],"thread":[206],"providing":[208],"thread.":[215],"These":[216],"threads":[218],"have":[219],"unique":[221],"form,":[224],"grouped":[227],"hash":[233],"function.":[234],"searching":[246],"optimized":[248],"hence":[249],"eager":[251],"strategy":[252],"implemented":[254],"hashing.":[260],"Proposed":[261],"system's":[262],"design":[263],"features":[264],"crawler":[266],"core":[267],"handling":[269],"main":[271],"crawling":[272],"tasks,":[273],"protocol":[277],"processing":[279],"modules.":[280],"greatest":[282],"positive":[283],"impact":[284],"occurred":[285],"when":[286],"there":[287],"was":[288],"pronounceable":[290],"change":[291],"system":[297],"compared":[299],"existing":[302],"process.":[303]},"counts_by_year":[],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}