{"id":"https://openalex.org/W7098364530","doi":null,"title":"Filtering near-duplicate documents","display_name":"Filtering near-duplicate documents","publication_year":1998,"publication_date":"1998-01-01","ids":{"openalex":"https://openalex.org/W7098364530"},"language":"en","primary_location":{"id":"pmh:oai:CiteSeerX.psu:10.1.1.295.8777","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.295.8777","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.cs.princeton.edu/courses/archive/spring05/cos598E/bib/CPM 2000.pdf","raw_type":"text"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108541542","display_name":"Andrei Broder","orcid":"https://orcid.org/0000-0001-7039-4170","observed_orcids":["https://orcid.org/0000-0001-7039-4170"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Andrei Z. Broder","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5108541542"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.56166667,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.9394999742507935,"subfield":{"id":"https://openalex.org/subfields/2614","display_name":"Theoretical Computer Science"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.9394999742507935,"subfield":{"id":"https://openalex.org/subfields/2614","display_name":"Theoretical Computer Science"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.0348999984562397,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.004399999976158142,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/near-duplicate-detection","display_name":"near-duplicate detection","score":0.890999972820282},{"id":"https://openalex.org/keywords/document-filtering","display_name":"document filtering","score":0.41499999165534973},{"id":"https://openalex.org/keywords/random-sampling","display_name":"random sampling","score":0.8610000014305115},{"id":"https://openalex.org/keywords/sketching","display_name":"sketching","score":0.49000000953674316},{"id":"https://openalex.org/keywords/web-indexing","display_name":"web indexing","score":0.8029999732971191},{"id":"https://openalex.org/keywords/threshold-testing","display_name":"threshold testing","score":0.5350000262260437}],"concepts":[{"id":"https://openalex.org/C2779231336","wikidata":"https://www.wikidata.org/wiki/Q7534724","display_name":"Sketch","level":2,"score":0.8120999932289124},{"id":"https://openalex.org/C64543145","wikidata":"https://www.wikidata.org/wiki/Q162942","display_name":"Intersection (aeronautics)","level":2,"score":0.6956999897956848},{"id":"https://openalex.org/C43364308","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Byte","level":2,"score":0.6389999985694885},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6360999941825867},{"id":"https://openalex.org/C75165309","wikidata":"https://www.wikidata.org/wiki/Q2258979","display_name":"Search engine indexing","level":2,"score":0.6215000152587891},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5634999871253967},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.5625},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5311999917030334},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.43149998784065247},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.42239999771118164},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38760000467300415},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3416999876499176},{"id":"https://openalex.org/C159254197","wikidata":"https://www.wikidata.org/wiki/Q1144915","display_name":"Lexicographical order","level":2,"score":0.33640000224113464},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3239000141620636},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.3100999891757965},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.28600001335144043},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.28459998965263367},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.28110000491142273},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.25429999828338623},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.2540999948978424}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:CiteSeerX.psu:10.1.1.295.8777","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.295.8777","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.cs.princeton.edu/courses/archive/spring05/cos598E/bib/CPM 2000.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2148885851","https://openalex.org/W2356740922","https://openalex.org/W2181426586","https://openalex.org/W7135892163","https://openalex.org/W7099024618","https://openalex.org/W7019408341","https://openalex.org/W3150251155","https://openalex.org/W2989913618","https://openalex.org/W2097184821","https://openalex.org/W7096093075"],"abstract_inverted_index":{"Abstract.":[0],"The":[1,15,125,168,187],"mathematical":[2],"concept":[3],"of":[4,12,33,37,41,45,48,84,118,150,157,170,175,177,184,211],"document":[5],"resemblance":[6,16,70,97,130,135],"captures":[7],"well":[8],"the":[9,39,46,68,96,147,172,180,204,209,212],"informal":[10],"notion":[11],"syntactic":[13],"similarity.":[14],"can":[17,111,161],"be":[18,112,162,183],"estimated":[19],"using":[20,114],"a":[21,30,49,100,115,139,155],"fixed":[22],"size":[23,40,149,174],"\u201csketch":[24],"\u201d":[25,117],"for":[26,56,128,165,189,203],"each":[27,166],"document.":[28,54,124,167],"For":[29],"large":[31,58],"collection":[32],"documents":[34,79,85,192],"(say":[35],"hundreds":[36],"millions)":[38],"this":[42,104,109],"sketch":[43],"is":[44,63,98,136,152],"order":[47],"few":[50],"hundred":[51],"bytes":[52,122],"per":[53,123],"However,":[55],"efficient":[57],"scale":[59],"web":[60],"indexing":[61],"it":[62,72,91],"not":[64],"necessary":[65],"to":[66,74,93],"determine":[67,75,94],"actual":[69],"value:":[71],"suffices":[73,92],"whether":[76,95],"newly":[77],"encountered":[78],"are":[80],"duplicates":[81],"or":[82],"near-duplicates":[83],"already":[86],"indexed.":[87],"In":[88,103],"other":[89],"words,":[90],"above":[99],"certain":[101],"threshold.":[102],"talk":[105],"we":[106],"show":[107],"how":[108],"determination":[110],"made":[113],"\u201dsample":[116],"less":[119],"than":[120],"50":[121],"basic":[126],"approach":[127],"computing":[129],"has":[131,195,200],"two":[132],"aspects:":[133],"first,":[134],"expressed":[137],"as":[138],"set":[140],"(of":[141],"strings)":[142],"intersection":[143,176],"problem,":[144],"and":[145,179,199],"second,":[146],"relative":[148,173],"intersections":[151],"evaluated":[153],"by":[154],"process":[156,169],"random":[158],"sampling":[159],"that":[160],"done":[163],"independently":[164],"estimating":[171],"sets":[178],"thresholdtestdiscussedabovecanbeappliedtoarbitrarysets,andthus":[181],"might":[182],"independent":[185],"interest.":[186],"algorithm":[188],"filtering":[190],"near-duplicate":[191],"discussed":[193],"here":[194],"been":[196,201],"successfully":[197],"implemented":[198],"used":[202],"last":[205],"three":[206],"years":[207],"in":[208],"context":[210],"AltaVista":[213],"search":[214],"engine.":[215],"1":[216]},"counts_by_year":[],"updated_date":"2026-10-14T16:39:01.820549","created_date":"2025-10-27T00:00:00"}