{"id":"https://openalex.org/W2110270499","doi":"https://doi.org/10.1145/1557019.1557166","title":"Incorporating site-level knowledge for incremental crawling of web forums","display_name":"Incorporating site-level knowledge for incremental crawling of web forums","publication_year":2009,"publication_date":"2009-06-28","ids":{"openalex":"https://openalex.org/W2110270499","doi":"https://doi.org/10.1145/1557019.1557166","mag":"2110270499"},"language":"en","primary_location":{"id":"doi:10.1145/1557019.1557166","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1557019.1557166","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th ACM SIGKDD international conference on Knowledge discovery and data mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002360117","display_name":"Jiang-Ming Yang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiang-Ming Yang","raw_affiliation_strings":["Microsoft Research, Asia, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Asia, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069272117","display_name":"Rui Cai","orcid":"https://orcid.org/0000-0002-6499-2091"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rui Cai","raw_affiliation_strings":["Microsoft Research, Asia, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Asia, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038762759","display_name":"Chunsong Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I135310074","display_name":"University of Wisconsin\u2013Madison","ror":"https://ror.org/01y2jtd41","country_code":"US","type":"education","lineage":["https://openalex.org/I135310074"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chunsong Wang","raw_affiliation_strings":["University of Wisconsin-Madison, Madison, WI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Wisconsin-Madison, Madison, WI, USA","institution_ids":["https://openalex.org/I135310074"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026978889","display_name":"Hua Huang","orcid":"https://orcid.org/0000-0002-2520-8938"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hua Huang","raw_affiliation_strings":["Beijing University of Posts and Telecommunications, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100433824","display_name":"Lei Zhang","orcid":"https://orcid.org/0000-0001-6926-0538"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Zhang","raw_affiliation_strings":["Microsoft Research, Asia, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Asia, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103733614","display_name":"Wei\u2010Ying Ma","orcid":"https://orcid.org/0000-0002-7384-0735"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei-Ying Ma","raw_affiliation_strings":["Microsoft Research, Asia, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Asia, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.435,"has_fulltext":false,"cited_by_count":20,"citation_normalized_percentile":{"value":0.95941127,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1375","last_page":"1384"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9900000095367432,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12285","display_name":"Surface Chemistry and Catalysis","score":0.9498000144958496,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/web-crawler","display_name":"Web crawler","score":0.9110510349273682},{"id":"https://openalex.org/keywords/crawling","display_name":"Crawling","score":0.8634480237960815},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8328741788864136},{"id":"https://openalex.org/keywords/timestamp","display_name":"Timestamp","score":0.7148805856704712},{"id":"https://openalex.org/keywords/thread","display_name":"Thread (computing)","score":0.6063084006309509},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.5575027465820312},{"id":"https://openalex.org/keywords/site-map","display_name":"Site map","score":0.5006661415100098},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4814854860305786},{"id":"https://openalex.org/keywords/web-page","display_name":"Web page","score":0.4366377592086792},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3328540325164795},{"id":"https://openalex.org/keywords/static-web-page","display_name":"Static web page","score":0.24653351306915283},{"id":"https://openalex.org/keywords/web-navigation","display_name":"Web navigation","score":0.1206483244895935},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.07097625732421875}],"concepts":[{"id":"https://openalex.org/C13743948","wikidata":"https://www.wikidata.org/wiki/Q45842","display_name":"Web crawler","level":2,"score":0.9110510349273682},{"id":"https://openalex.org/C100368936","wikidata":"https://www.wikidata.org/wiki/Q1411725","display_name":"Crawling","level":2,"score":0.8634480237960815},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8328741788864136},{"id":"https://openalex.org/C113954288","wikidata":"https://www.wikidata.org/wiki/Q186885","display_name":"Timestamp","level":2,"score":0.7148805856704712},{"id":"https://openalex.org/C138101251","wikidata":"https://www.wikidata.org/wiki/Q213092","display_name":"Thread (computing)","level":2,"score":0.6063084006309509},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.5575027465820312},{"id":"https://openalex.org/C67617509","wikidata":"https://www.wikidata.org/wiki/Q1503327","display_name":"Site map","level":5,"score":0.5006661415100098},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4814854860305786},{"id":"https://openalex.org/C21959979","wikidata":"https://www.wikidata.org/wiki/Q36774","display_name":"Web page","level":2,"score":0.4366377592086792},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3328540325164795},{"id":"https://openalex.org/C173576120","wikidata":"https://www.wikidata.org/wiki/Q2641220","display_name":"Static web page","level":4,"score":0.24653351306915283},{"id":"https://openalex.org/C61096286","wikidata":"https://www.wikidata.org/wiki/Q7978592","display_name":"Web navigation","level":3,"score":0.1206483244895935},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.07097625732421875},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C105702510","wikidata":"https://www.wikidata.org/wiki/Q514","display_name":"Anatomy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/1557019.1557166","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1557019.1557166","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th ACM SIGKDD international conference on Knowledge discovery and data mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W1489992655","https://openalex.org/W2016122268","https://openalex.org/W2018928332","https://openalex.org/W2023094000","https://openalex.org/W2030453570","https://openalex.org/W2038378248","https://openalex.org/W2059713800","https://openalex.org/W2099126271","https://openalex.org/W2100779529","https://openalex.org/W2105803339","https://openalex.org/W2113184419","https://openalex.org/W2124673015","https://openalex.org/W2127536142","https://openalex.org/W2140077965","https://openalex.org/W2151932833","https://openalex.org/W2162502927","https://openalex.org/W2168358004","https://openalex.org/W3149154678","https://openalex.org/W6654369015"],"related_works":["https://openalex.org/W2084213350","https://openalex.org/W2566658409","https://openalex.org/W3119324922","https://openalex.org/W2352686120","https://openalex.org/W2372594123","https://openalex.org/W2358310581","https://openalex.org/W2364692302","https://openalex.org/W1935121990","https://openalex.org/W1506122440","https://openalex.org/W2371838993"],"abstract_inverted_index":{"We":[0],"study":[1],"in":[2,21,50,157],"this":[3,142],"paper":[4],"the":[5,31,56,81,91,102,106,133,136],"problem":[6],"of":[7,10,34,55,66,159,179,185],"incremental":[8],"crawling":[9,51],"web":[11,23],"forums,":[12],"which":[13,149],"is":[14,47,87,150],"a":[15,74,97,128,170],"very":[16],"fundamental":[17],"yet":[18],"challenging":[19],"step":[20],"many":[22],"applications.":[24],"Traditional":[25],"approaches":[26],"mainly":[27],"focus":[28],"on":[29,111,141,188],"scheduling":[30],"revisiting":[32],"strategy":[33,76],"each":[35,68,124],"individual":[36,45,69],"page.":[37],"However,":[38],"simply":[39],"assigning":[40],"different":[41,44,57],"weights":[42],"for":[43,96,123],"pages":[46,113],"usually":[48],"inefficient":[49],"forum":[52,60,99],"sites":[53,61],"because":[54],"characteristics":[58],"between":[59],"and":[62,164,183],"general":[63],"websites.":[64],"Instead":[65],"treating":[67],"page":[70],"independently,":[71],"we":[72,126,144],"propose":[73],"list-wise":[75],"by":[77],"taking":[78],"into":[79],"account":[80],"site-level":[82,85],"knowledge.":[83],"Such":[84],"knowledge":[86],"mined":[88],"through":[89],"reconstructing":[90],"linking":[92],"structure,":[93],"called":[94],"sitemap,":[95,103],"given":[98],"site.":[100],"With":[101],"posts":[104],"from":[105],"same":[107],"thread":[108],"but":[109],"distributed":[110],"various":[112,190],"can":[114],"be":[115],"concatenated":[116],"according":[117],"to":[118,131],"their":[119],"timestamps.":[120],"After":[121],"that,":[122],"thread,":[125],"employ":[127],"regression":[129],"model":[130],"predict":[132],"time":[134],"when":[135],"next":[137],"post":[138],"arrives.":[139],"Based":[140],"model,":[143],"develop":[145],"an":[146],"efficient":[147],"crawler":[148,167,187],"260%":[151],"faster":[152],"than":[153],"some":[154],"state-of-the-art":[155],"methods":[156],"terms":[158],"fetching":[160],"new":[161],"generated":[162],"content;":[163],"meanwhile":[165],"our":[166,186],"also":[168],"ensure":[169],"high":[171],"coverage":[172],"ratio.":[173],"Experimental":[174],"results":[175],"show":[176],"promising":[177],"performance":[178],"Coverage,":[180],"Bandwidth":[181],"utilization,":[182],"Timeliness":[184],"18":[189],"forums.":[191]},"counts_by_year":[{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":2},{"year":2014,"cited_by_count":2},{"year":2013,"cited_by_count":3},{"year":2012,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}