{"id":"https://openalex.org/W198700214","doi":null,"title":"SpotSigs: Near Duplicate Detection in Web Page Collections","display_name":"SpotSigs: Near Duplicate Detection in Web Page Collections","publication_year":2007,"publication_date":"2007-01-01","ids":{"openalex":"https://openalex.org/W198700214","mag":"198700214"},"language":"en","primary_location":{"id":"mag:198700214","doi":null,"is_oa":false,"landing_page_url":"http://dbpubs.stanford.edu:8090/pub/2007-37","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"volume":null,"type":"dissertation","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055950945","display_name":"Jonathan Siddharth","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jonathan Siddharth","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5071755113","display_name":"Andreas Paepcke","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Andreas Paepcke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.8866999745368958,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.8866999745368958,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.07320000231266022,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.005400000140070915,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/near-duplicate-detection","display_name":"near-duplicate detection","score":0.9679999947547913},{"id":"https://openalex.org/keywords/web-page-collection","display_name":"web page collection","score":0.7239999771118164},{"id":"https://openalex.org/keywords/hash-collisions","display_name":"hash collisions","score":0.7200000286102295},{"id":"https://openalex.org/keywords/news-archives","display_name":"news archives","score":0.6079999804496765},{"id":"https://openalex.org/keywords/political-science-research","display_name":"political science research","score":0.5649999976158142},{"id":"https://openalex.org/keywords/web-archives","display_name":"web archives","score":0.41499999165534973}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7375822067260742},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.7180320620536804},{"id":"https://openalex.org/C21959979","wikidata":"https://www.wikidata.org/wiki/Q36774","display_name":"Web page","level":2,"score":0.6236370205879211},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.6149808168411255},{"id":"https://openalex.org/C99138194","wikidata":"https://www.wikidata.org/wiki/Q183427","display_name":"Hash function","level":2,"score":0.606162428855896},{"id":"https://openalex.org/C88006597","wikidata":"https://www.wikidata.org/wiki/Q690117","display_name":"Disk formatting","level":2,"score":0.6003494262695312},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5554797053337097},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.4164767861366272},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.4061993360519409},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:198700214","doi":null,"is_oa":false,"landing_page_url":"http://dbpubs.stanford.edu:8090/pub/2007-37","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W1575770195","https://openalex.org/W1596691921","https://openalex.org/W1609518033","https://openalex.org/W1941707302","https://openalex.org/W2007842132","https://openalex.org/W2012833704","https://openalex.org/W2026968007","https://openalex.org/W2067432306","https://openalex.org/W2085922539","https://openalex.org/W2111549955","https://openalex.org/W2148885851","https://openalex.org/W2152565070","https://openalex.org/W2153180170","https://openalex.org/W2164634022"],"related_works":["https://openalex.org/W1591455229","https://openalex.org/W2109803107","https://openalex.org/W153295694","https://openalex.org/W2136774890","https://openalex.org/W7096093075","https://openalex.org/W2181426586","https://openalex.org/W2085922539","https://openalex.org/W7019408341","https://openalex.org/W7099493317","https://openalex.org/W2008577673"],"abstract_inverted_index":{"Motivated":[0],"by":[1,92],"our":[2,108,161],"work":[3],"with":[4,28],"political":[5],"scientists":[6,17],"we":[7,180],"present":[8],"an":[9],"algorithm":[10,77,118],"that":[11,37,80],"detects":[12,119],"near-duplicate":[13],"Web":[14,19,152],"pages.":[15],"These":[16],"analyze":[18],"archives":[20,25],"of":[21,35,62,71,95,110,124,144,150,160,173,204,210],"news":[22,51],"sites.":[23],"The":[24,59],"were":[26],"collected":[27],"crawlers":[29],"and":[30,100,136,141,154,176,188,207],"contain":[31],"a":[32,147,202,208],"large":[33],"number":[34],"pages":[36,55,67],"look":[38],"very":[39],"different":[40,131],"because":[41],"the":[42,50,54,66,69,93,103,112,117,122,130,156,171,194],"frame":[43],"around":[44],"their":[45],"core":[46],"content":[47,72],"differs.":[48],"However,":[49],"stories":[52],"in":[53,102],"are":[56,81,90],"nearly":[57],"identical.":[58],"close":[60],"proximity":[61],"unrelated":[63],"items":[64],"on":[65,146],"makes":[68],"detection":[70,172],"overlap":[73,120],"difficult.":[74],"Our":[75],"SpotSigs":[76,132,145,200],"generates":[78],"signatures":[79,89,123],"spread":[82],"across":[83],"each":[84],"document.":[85],"Places":[86],"for":[87,170,213],"these":[88],"determined":[91],"placement":[94],"common":[96],"words,":[97],"like":[98],"'is'":[99],"'the'":[101],"documents.":[104,195],"We":[105,127,139],"can":[106],"vary":[107],"method":[109],"computing":[111],"signatures.":[113],"Using":[114],"hash":[115],"collisions":[116],"among":[121],"matching":[125],"contents.":[126],"study":[128,155],"how":[129],"parameters":[133],"impact":[134],"precision":[135,203],"recall":[137,209],"performance.":[138],"propose":[140],"evaluate":[142],"variants":[143],"test":[148],"bed":[149],"2168":[151],"Pages":[153],"tradeoffs":[157],"involved.":[158],"One":[159],"motivations":[162],"was":[163],"also":[164],"to":[165,177],"keep":[166],"pre-processing":[167],"requirements":[168],"low":[169],"near":[174,214],"duplicates":[175],"this":[178,197],"end":[179],"do":[181],"not":[182],"remove":[183],"ads,":[184],"client":[185],"side":[186],"scripts":[187],"other":[189],"HTML":[190],"formatting":[191],"elements":[192],"from":[193],"On":[196],"data":[198],"set":[199],"obtains":[201],"over":[205,211],"93%":[206],"85%":[212],"duplicate":[215],"detection.":[216]},"counts_by_year":[],"updated_date":"2025-11-06T04:12:42.849631","created_date":"2016-06-24T00:00:00"}