{"id":"https://openalex.org/W2017224880","doi":"https://doi.org/10.1145/1095872.1095875","title":"Learning to crawl","display_name":"Learning to crawl","publication_year":2005,"publication_date":"2005-10-01","ids":{"openalex":"https://openalex.org/W2017224880","doi":"https://doi.org/10.1145/1095872.1095875","mag":"2017224880"},"language":"en","primary_location":{"id":"doi:10.1145/1095872.1095875","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1095872.1095875","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5038050803","display_name":"Gautam Pant","orcid":"https://orcid.org/0000-0001-7414-2325"},"institutions":[{"id":"https://openalex.org/I223532165","display_name":"University of Utah","ror":"https://ror.org/03r0ha626","country_code":"US","type":"education","lineage":["https://openalex.org/I223532165"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gautam Pant","raw_affiliation_strings":["The University of Utah, Salt Lake City, UT"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Utah, Salt Lake City, UT","institution_ids":["https://openalex.org/I223532165"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016839456","display_name":"Padmini Srinivasan","orcid":"https://orcid.org/0000-0001-8791-403X"},"institutions":[{"id":"https://openalex.org/I126307644","display_name":"University of Iowa","ror":"https://ror.org/036jqmy94","country_code":"US","type":"education","lineage":["https://openalex.org/I126307644"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Padmini Srinivasan","raw_affiliation_strings":["The University of Iowa, Iowa City, IA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Iowa, Iowa City, IA","institution_ids":["https://openalex.org/I126307644"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":19.5186,"has_fulltext":false,"cited_by_count":142,"citation_normalized_percentile":{"value":0.98989975,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"23","issue":"4","first_page":"430","last_page":"462"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9851999878883362,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10538","display_name":"Data Mining Algorithms and Applications","score":0.9650999903678894,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/web-crawler","display_name":"Web crawler","score":0.8638256788253784},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7991570234298706},{"id":"https://openalex.org/keywords/crawling","display_name":"Crawling","score":0.7348965406417847},{"id":"https://openalex.org/keywords/naive-bayes-classifier","display_name":"Naive Bayes classifier","score":0.6535094380378723},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6348544359207153},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5627444386482239},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.4479557275772095},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4442221522331238},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.4122179448604584},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.410244882106781},{"id":"https://openalex.org/keywords/support-vector-machine","display_name":"Support vector machine","score":0.3707394301891327},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.22277477383613586},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.13464894890785217}],"concepts":[{"id":"https://openalex.org/C13743948","wikidata":"https://www.wikidata.org/wiki/Q45842","display_name":"Web crawler","level":2,"score":0.8638256788253784},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7991570234298706},{"id":"https://openalex.org/C100368936","wikidata":"https://www.wikidata.org/wiki/Q1411725","display_name":"Crawling","level":2,"score":0.7348965406417847},{"id":"https://openalex.org/C52001869","wikidata":"https://www.wikidata.org/wiki/Q812530","display_name":"Naive Bayes classifier","level":3,"score":0.6535094380378723},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6348544359207153},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5627444386482239},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.4479557275772095},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4442221522331238},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.4122179448604584},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.410244882106781},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.3707394301891327},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.22277477383613586},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.13464894890785217},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C105702510","wikidata":"https://www.wikidata.org/wiki/Q514","display_name":"Anatomy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/1095872.1095875","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1095872.1095875","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":90,"referenced_works":["https://openalex.org/W55136975","https://openalex.org/W160262995","https://openalex.org/W179016587","https://openalex.org/W198953946","https://openalex.org/W1485536830","https://openalex.org/W1489992655","https://openalex.org/W1490760466","https://openalex.org/W1496929357","https://openalex.org/W1503924817","https://openalex.org/W1512098439","https://openalex.org/W1524645404","https://openalex.org/W1540371141","https://openalex.org/W1540550673","https://openalex.org/W1550206324","https://openalex.org/W1557757161","https://openalex.org/W1604938182","https://openalex.org/W1679844312","https://openalex.org/W1782225848","https://openalex.org/W1912123407","https://openalex.org/W1924689489","https://openalex.org/W1956559956","https://openalex.org/W1995341919","https://openalex.org/W1999991474","https://openalex.org/W2001832505","https://openalex.org/W2002016471","https://openalex.org/W2003000928","https://openalex.org/W2004436106","https://openalex.org/W2005374071","https://openalex.org/W2005579877","https://openalex.org/W2014134732","https://openalex.org/W2016926633","https://openalex.org/W2017726337","https://openalex.org/W2042264548","https://openalex.org/W2042330455","https://openalex.org/W2044743392","https://openalex.org/W2045998703","https://openalex.org/W2048538797","https://openalex.org/W2066814510","https://openalex.org/W2098162425","https://openalex.org/W2101057528","https://openalex.org/W2108995755","https://openalex.org/W2116424792","https://openalex.org/W2121294789","https://openalex.org/W2124673015","https://openalex.org/W2128915886","https://openalex.org/W2129476886","https://openalex.org/W2137983211","https://openalex.org/W2139212933","https://openalex.org/W2141825421","https://openalex.org/W2144242878","https://openalex.org/W2147164982","https://openalex.org/W2147947791","https://openalex.org/W2151007976","https://openalex.org/W2154642048","https://openalex.org/W2156909104","https://openalex.org/W2156963321","https://openalex.org/W2162502927","https://openalex.org/W2162630660","https://openalex.org/W2170541679","https://openalex.org/W2323675077","https://openalex.org/W2325227998","https://openalex.org/W2435251607","https://openalex.org/W2490800645","https://openalex.org/W2560674852","https://openalex.org/W2606748186","https://openalex.org/W2615497679","https://openalex.org/W2742161170","https://openalex.org/W2766736793","https://openalex.org/W2768581363","https://openalex.org/W2799061466","https://openalex.org/W2902404196","https://openalex.org/W2911969827","https://openalex.org/W3029645440","https://openalex.org/W3141221564","https://openalex.org/W3146803896","https://openalex.org/W3193477162","https://openalex.org/W4206686222","https://openalex.org/W4235537826","https://openalex.org/W4237452273","https://openalex.org/W4246519874","https://openalex.org/W4249987776","https://openalex.org/W4293775970","https://openalex.org/W6602137617","https://openalex.org/W6632223008","https://openalex.org/W6640862754","https://openalex.org/W6651560827","https://openalex.org/W6653787752","https://openalex.org/W6661021596","https://openalex.org/W6683581212","https://openalex.org/W6738852829"],"related_works":["https://openalex.org/W2566658409","https://openalex.org/W3119324922","https://openalex.org/W2352686120","https://openalex.org/W2372594123","https://openalex.org/W2358310581","https://openalex.org/W2964752624","https://openalex.org/W2026132847","https://openalex.org/W4385695127","https://openalex.org/W2137810919","https://openalex.org/W4255854114"],"abstract_inverted_index":{"Topical":[0],"crawling":[1,69],"is":[2,71,108],"a":[3,74,79,109,114],"young":[4],"and":[5],"creative":[6],"area":[7],"of":[8,14,24,64,98,130,140],"research":[9],"that":[10,105,149],"holds":[11],"the":[12,36,50,83,90,99,127,159],"promise":[13],"benefiting":[15],"from":[16,53],"several":[17],"sophisticated":[18],"data":[19],"mining":[20],"techniques.":[21],"The":[22,68,85],"use":[23],"classification":[25,66],"algorithms":[26],"to":[27,89],"guide":[28],"topical":[29,115,154],"crawlers":[30,155],"has":[31,42],"been":[32,43],"sporadically":[33],"suggested":[34],"in":[35],"literature.":[37],"No":[38],"systematic":[39],"study,":[40],"however,":[41],"done":[44],"on":[45,158],"their":[46],"relative":[47],"merits.":[48],"Using":[49],"lessons":[51],"learned":[52],"our":[54],"previous":[55],"crawler":[56,91,116],"evaluation":[57],"studies,":[58],"we":[59],"experiment":[60],"with":[61,119,161],"multiple":[62],"versions":[63],"different":[65,153],"schemes.":[67],"process":[70],"modeled":[72],"as":[73],"parallel":[75],"best-first":[76],"search":[77],"over":[78],"graph":[80],"defined":[81],"by":[82,137,144],"Web.":[84],"classifiers":[86],"provide":[87],"heuristics":[88],"thus":[92],"biasing":[93],"it":[94],"towards":[95],"certain":[96],"portions":[97],"Web":[100,160],"graph.":[101],"Our":[102],"results":[103],"show":[104],"Naive":[106,131],"Bayes":[107,132],"weak":[110,128],"choice":[111],"for":[112],"guiding":[113],"when":[117],"compared":[118],"Support":[120],"Vector":[121],"Machine":[122],"or":[123],"Neural":[124],"Network.":[125],"Further,":[126],"performance":[129],"can":[133],"be":[134],"partly":[135],"explained":[136],"extreme":[138],"skewness":[139],"posterior":[141],"probabilities":[142],"generated":[143],"it.":[145],"We":[146],"also":[147],"observe":[148],"despite":[150],"similar":[151],"performances,":[152],"cover":[156],"subspaces":[157],"low":[162],"overlap.":[163]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":11},{"year":2016,"cited_by_count":8},{"year":2015,"cited_by_count":6},{"year":2014,"cited_by_count":16},{"year":2013,"cited_by_count":16},{"year":2012,"cited_by_count":11}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2016-06-24T00:00:00"}