{"id":"https://openalex.org/W7163324196","doi":"https://doi.org/10.48550/arxiv.2606.03001","title":"FOLD: Fuzzy Online Deduplication for Very Large Evolving Datasets via Approximate Nearest Neighbor Search","display_name":"FOLD: Fuzzy Online Deduplication for Very Large Evolving Datasets via Approximate Nearest Neighbor Search","publication_year":2026,"publication_date":"2026-06-02","ids":{"openalex":"https://openalex.org/W7163324196","doi":"https://doi.org/10.48550/arxiv.2606.03001"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.03001","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.03001","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.03001","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5115736649","display_name":"Nelson Bore","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bore, Nelson","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137735696","display_name":"Pritish Mishra","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mishra, Pritish","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137777001","display_name":"Constantin Adam","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Adam, Constantin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072600785","display_name":"Eyal de Lara","orcid":"https://orcid.org/0000-0002-6730-555X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"de Lara, Eyal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5055620336","display_name":"Oana Balmau","orcid":"https://orcid.org/0000-0002-6822-8891"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Balmau, Oana","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.19589999318122864,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.19589999318122864,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09489999711513519,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.09059999883174896,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-deduplication","display_name":"Data deduplication","score":0.8269000053405762},{"id":"https://openalex.org/keywords/jaccard-index","display_name":"Jaccard index","score":0.7448999881744385},{"id":"https://openalex.org/keywords/bitmap","display_name":"Bitmap","score":0.7073000073432922},{"id":"https://openalex.org/keywords/tree-traversal","display_name":"Tree traversal","score":0.5565999746322632},{"id":"https://openalex.org/keywords/fuzzy-logic","display_name":"Fuzzy logic","score":0.4390000104904175},{"id":"https://openalex.org/keywords/hash-function","display_name":"Hash function","score":0.4359000027179718},{"id":"https://openalex.org/keywords/precision-and-recall","display_name":"Precision and recall","score":0.40950000286102295},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.36899998784065247},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.35740000009536743}],"concepts":[{"id":"https://openalex.org/C32587265","wikidata":"https://www.wikidata.org/wiki/Q1182260","display_name":"Data deduplication","level":2,"score":0.8269000053405762},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7766000032424927},{"id":"https://openalex.org/C203519979","wikidata":"https://www.wikidata.org/wiki/Q865360","display_name":"Jaccard index","level":3,"score":0.7448999881744385},{"id":"https://openalex.org/C3115412","wikidata":"https://www.wikidata.org/wiki/Q1194708","display_name":"Bitmap","level":2,"score":0.7073000073432922},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5983999967575073},{"id":"https://openalex.org/C140745168","wikidata":"https://www.wikidata.org/wiki/Q1210082","display_name":"Tree traversal","level":2,"score":0.5565999746322632},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.478300005197525},{"id":"https://openalex.org/C58166","wikidata":"https://www.wikidata.org/wiki/Q224821","display_name":"Fuzzy logic","level":2,"score":0.4390000104904175},{"id":"https://openalex.org/C99138194","wikidata":"https://www.wikidata.org/wiki/Q183427","display_name":"Hash function","level":2,"score":0.4359000027179718},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.40950000286102295},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.36899998784065247},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.35740000009536743},{"id":"https://openalex.org/C116738811","wikidata":"https://www.wikidata.org/wiki/Q608751","display_name":"Nearest neighbor search","level":2,"score":0.3181000053882599},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3147999942302704},{"id":"https://openalex.org/C34736171","wikidata":"https://www.wikidata.org/wiki/Q918333","display_name":"Preprocessor","level":2,"score":0.3100000023841858},{"id":"https://openalex.org/C43364308","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Byte","level":2,"score":0.30720001459121704},{"id":"https://openalex.org/C67388219","wikidata":"https://www.wikidata.org/wiki/Q207440","display_name":"Hash table","level":3,"score":0.302700012922287},{"id":"https://openalex.org/C96333769","wikidata":"https://www.wikidata.org/wiki/Q907955","display_name":"Graph traversal","level":3,"score":0.30239999294281006},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.2879999876022339},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.28369998931884766},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.2833999991416931},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.2815999984741211},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2806999981403351},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.2782999873161316},{"id":"https://openalex.org/C74270461","wikidata":"https://www.wikidata.org/wiki/Q1625299","display_name":"Locality-sensitive hashing","level":4,"score":0.27000001072883606},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.2653999924659729},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.25999999046325684},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.2578999996185303},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.25}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.03001","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.03001","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.03001","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.03001","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","score":0.7398871183395386,"id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Fuzzy":[0],"deduplication":[1,79,124],"is":[2,31,119,138,152],"key":[3],"to":[4,25,126,130,141,179,238,245],"constructing":[5],"large":[6,59],"language":[7],"model":[8],"training":[9],"corpora.":[10],"However,":[11,136],"classic":[12],"Locality-Sensitive":[13],"Hashing":[14],"(LSH)":[15],"pipelines":[16],"scale":[17],"poorly":[18],"as":[19,133],"corpora":[20],"grow":[21],"and":[22,61,86,217,242],"are":[23],"ill-suited":[24],"continuous":[26],"ingestion.":[27],"The":[28,149],"main":[29],"issue":[30,151],"that":[32,81,200],"each":[33,54,108],"new":[34],"document":[35,110],"batch":[36],"must":[37,62],"be":[38],"checked":[39],"against":[40],"the":[41,47,50,64,115,120,153,161,222],"admitted":[42,99],"corpus":[43,48],"before":[44],"insertion.":[45],"As":[46],"grows,":[49],"LSH":[51,240],"buckets":[52,60],"grow:":[53],"query":[55],"can":[56],"hit":[57],"several":[58],"scan":[63],"returned":[65],"candidates.":[66],"To":[67],"solve":[68],"this":[69,195],"problem,":[70],"we":[71],"present":[72],"RAD":[73,91,118,193,220],"(Retrieval-Augmented":[74],"Deduplication),":[75],"an":[76,93,174,246],"online":[77,122,227],"fuzzy":[78,123,165,228],"system":[80,125],"delivers":[82,243],"both":[83],"high":[84,143],"recall":[85,144,169,236],"throughput":[87,132,248],"for":[88,107,164,226],"evolving":[89],"datasets.":[90],"maintains":[92,234],"incrementally":[94],"updated":[95],"HNSW":[96,175,208],"index":[97],"over":[98],"documents,":[100,232],"retrieving":[101],"a":[102,188,197,202],"small,":[103],"high-quality":[104],"candidate":[105],"neighborhood":[106],"incoming":[109],"instead":[111],"of":[112,191],"repeatedly":[113],"re-scanning":[114],"accumulated":[116],"corpus.":[117],"first":[121],"use":[127],"HNSW,":[128],"leading":[129],"stable":[131],"datasets":[134,213],"grow.":[135],"it":[137,233],"not":[139],"easy":[140],"maintain":[142],"when":[145,170],"using":[146],"HNSW-style":[147],"indexes.":[148],"core":[150],"distance":[154,180],"metric":[155,162],"between":[156],"graph":[157,184],"nodes.":[158],"Jaccard":[159],"similarity,":[160],"used":[163],"deduplication,":[166],"yields":[167],"low":[168],"applied":[171],"out-of-the-box":[172],"with":[173,196],"index.":[176],"It":[177],"leads":[178],"score":[181],"crowding,":[182],"making":[183],"traversal":[185],"unreliable":[186],"within":[187],"bounded":[189],"number":[190],"steps.":[192],"addresses":[194],"bitmap":[198],"representation":[199],"provides":[201],"more":[203],"discriminative,":[204],"Jaccard-aligned":[205],"signal":[206],"during":[207],"search.":[209],"Across":[210],"four":[211],"LLM-scale":[212],"(LM1B,":[214],"C4,":[215],"RealNews,":[216],"Common":[218],"Crawl),":[219],"preserves":[221],"scaling":[223],"trajectory":[224],"needed":[225],"deduplication:":[229],"at":[230],"30M":[231],"0.94-0.97":[235],"relative":[237],"state-of-the-art":[239],"solutions,":[241],"up":[244],"8x":[247],"increase.":[249]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-04T00:00:00"}