{"id":"https://openalex.org/W2007993754","doi":"https://doi.org/10.1109/bigdata.2013.6691610","title":"Massively scalable near duplicate detection in streams of documents using MDSH","display_name":"Massively scalable near duplicate detection in streams of documents using MDSH","publication_year":2013,"publication_date":"2013-10-01","ids":{"openalex":"https://openalex.org/W2007993754","doi":"https://doi.org/10.1109/bigdata.2013.6691610","mag":"2007993754"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata.2013.6691610","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata.2013.6691610","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2013 IEEE International Conference on Big Data","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036207141","display_name":"Paul Logasa Bogen","orcid":"https://orcid.org/0000-0002-4009-7789"},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Paul Logasa Bogen","raw_affiliation_strings":["Computational Data Analytics Group, Oak Ridge National Laboratory, Oak Ridge, TN","Comput. Data Analytics Group, Oak Ridge Nat. Lab., Oak Ridge, TN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computational Data Analytics Group, Oak Ridge National Laboratory, Oak Ridge, TN","institution_ids":["https://openalex.org/I1289243028"]},{"raw_affiliation_string":"Comput. Data Analytics Group, Oak Ridge Nat. Lab., Oak Ridge, TN, USA","institution_ids":["https://openalex.org/I1289243028"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055365897","display_name":"Christopher T. Symons","orcid":null},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christopher T. Symons","raw_affiliation_strings":["Computational Data Analytics Group, Oak Ridge National Laboratory, Oak Ridge, TN","Comput. Data Analytics Group, Oak Ridge Nat. Lab., Oak Ridge, TN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computational Data Analytics Group, Oak Ridge National Laboratory, Oak Ridge, TN","institution_ids":["https://openalex.org/I1289243028"]},{"raw_affiliation_string":"Comput. Data Analytics Group, Oak Ridge Nat. Lab., Oak Ridge, TN, USA","institution_ids":["https://openalex.org/I1289243028"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016858324","display_name":"Amber McKenzie","orcid":null},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amber McKenzie","raw_affiliation_strings":["Computational Data Analytics Group, Oak Ridge National Laboratory, Oak Ridge, TN","Comput. Data Analytics Group, Oak Ridge Nat. Lab., Oak Ridge, TN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computational Data Analytics Group, Oak Ridge National Laboratory, Oak Ridge, TN","institution_ids":["https://openalex.org/I1289243028"]},{"raw_affiliation_string":"Comput. Data Analytics Group, Oak Ridge Nat. Lab., Oak Ridge, TN, USA","institution_ids":["https://openalex.org/I1289243028"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021986280","display_name":"Robert M. Patton","orcid":"https://orcid.org/0000-0002-8101-0571"},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Robert M. Patton","raw_affiliation_strings":["Computational Data Analytics Group, Oak Ridge National Laboratory, Oak Ridge, TN","Comput. Data Analytics Group, Oak Ridge Nat. Lab., Oak Ridge, TN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computational Data Analytics Group, Oak Ridge National Laboratory, Oak Ridge, TN","institution_ids":["https://openalex.org/I1289243028"]},{"raw_affiliation_string":"Comput. Data Analytics Group, Oak Ridge Nat. Lab., Oak Ridge, TN, USA","institution_ids":["https://openalex.org/I1289243028"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063942604","display_name":"Rob Gillen","orcid":"https://orcid.org/0000-0002-9002-1440"},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Robert E. Gillen","raw_affiliation_strings":["Computational Data Analytics Group, Oak Ridge National Laboratory, Oak Ridge, TN","Comput. Data Analytics Group, Oak Ridge Nat. Lab., Oak Ridge, TN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computational Data Analytics Group, Oak Ridge National Laboratory, Oak Ridge, TN","institution_ids":["https://openalex.org/I1289243028"]},{"raw_affiliation_string":"Comput. Data Analytics Group, Oak Ridge Nat. Lab., Oak Ridge, TN, USA","institution_ids":["https://openalex.org/I1289243028"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1289243028"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.11617709,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"18","issue":null,"first_page":"480","last_page":"486"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.875526487827301},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.757760763168335},{"id":"https://openalex.org/keywords/hash-function","display_name":"Hash function","score":0.6764596104621887},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.617038369178772},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5323866605758667},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.42966705560684204},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3280913829803467},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.2690948247909546}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.875526487827301},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.757760763168335},{"id":"https://openalex.org/C99138194","wikidata":"https://www.wikidata.org/wiki/Q183427","display_name":"Hash function","level":2,"score":0.6764596104621887},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.617038369178772},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5323866605758667},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.42966705560684204},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3280913829803467},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.2690948247909546},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata.2013.6691610","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata.2013.6691610","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2013 IEEE International Conference on Big Data","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5400000214576721,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":32,"referenced_works":["https://openalex.org/W35739717","https://openalex.org/W189214596","https://openalex.org/W191584165","https://openalex.org/W338913944","https://openalex.org/W1489021533","https://openalex.org/W1504644433","https://openalex.org/W1507273245","https://openalex.org/W1516380833","https://openalex.org/W1522405883","https://openalex.org/W1814571536","https://openalex.org/W1994142851","https://openalex.org/W2007968818","https://openalex.org/W2041933468","https://openalex.org/W2049589130","https://openalex.org/W2074769045","https://openalex.org/W2075672964","https://openalex.org/W2088088007","https://openalex.org/W2110166424","https://openalex.org/W2115054880","https://openalex.org/W2122457239","https://openalex.org/W2138399774","https://openalex.org/W2145349611","https://openalex.org/W2150731624","https://openalex.org/W2162340532","https://openalex.org/W2171656596","https://openalex.org/W2250175843","https://openalex.org/W2293597654","https://openalex.org/W2571012312","https://openalex.org/W6607590210","https://openalex.org/W6607669822","https://openalex.org/W6669448905","https://openalex.org/W6691475363"],"related_works":["https://openalex.org/W2389214306","https://openalex.org/W4235240664","https://openalex.org/W2965083567","https://openalex.org/W1838576100","https://openalex.org/W2095886385","https://openalex.org/W2889616422","https://openalex.org/W2089704382","https://openalex.org/W1983399550","https://openalex.org/W97075385","https://openalex.org/W2357523926"],"abstract_inverted_index":{"In":[0,79],"a":[1,24],"world":[2],"where":[3],"large-scale":[4],"text":[5,100],"collections":[6],"are":[7,14,22],"not":[8],"only":[9,49],"becoming":[10,23],"ubiquitous":[11],"but":[12],"also":[13],"growing":[15,25,97],"at":[16],"increasing":[17],"rates,":[18],"near":[19],"duplicate":[20],"documents":[21],"concern":[26],"that":[27],"has":[28],"the":[29,63,66,104,118,121],"potential":[30],"to":[31,42],"hinder":[32],"many":[33],"different":[34],"information":[35],"filtering":[36],"tasks.":[37],"While":[38],"others":[39],"have":[40,48],"tried":[41],"address":[43],"this":[44,80],"problem,":[45],"prior":[46],"techniques":[47,111],"been":[50],"used":[51],"on":[52,86,94],"limited":[53],"collection":[54],"sizes":[55],"and":[56,106,112],"static":[57],"cases.":[58],"We":[59,102],"will":[60],"briefly":[61],"describe":[62],"problem":[64],"in":[65],"context":[67],"of":[68,99,109,117,120],"Open":[69],"Source":[70],"analysis":[71,116],"along":[72],"with":[73],"our":[74,110,126],"additional":[75],"constraints":[76],"for":[77,92],"performance.":[78],"work":[81],"we":[82],"propose":[83],"two":[84],"variations":[85],"Multi-dimensional":[87],"Spectral":[88],"Hash":[89],"(MDSH)":[90],"tailored":[91],"working":[93],"extremely":[95],"large,":[96],"sets":[98],"documents.":[101],"analyze":[103],"memory":[105],"runtime":[107],"characteristics":[108],"provide":[113],"an":[114],"informal":[115],"quality":[119],"near-duplicate":[122],"clusters":[123],"produced":[124],"by":[125],"techniques.":[127]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}