{"id":"https://openalex.org/W4406458249","doi":"https://doi.org/10.1109/bigdata62323.2024.10825314","title":"Common Pitfalls with Data Deduplication Parameters and Metrics","display_name":"Common Pitfalls with Data Deduplication Parameters and Metrics","publication_year":2024,"publication_date":"2024-12-15","ids":{"openalex":"https://openalex.org/W4406458249","doi":"https://doi.org/10.1109/bigdata62323.2024.10825314"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata62323.2024.10825314","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10825314","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Luke Schultz","orcid":null},"institutions":[{"id":"https://openalex.org/I154425047","display_name":"University of Alberta","ror":"https://ror.org/0160cpw27","country_code":"CA","type":"education","lineage":["https://openalex.org/I154425047"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Luke Schultz","raw_affiliation_strings":["University of Alberta"],"affiliations":[{"raw_affiliation_string":"University of Alberta","institution_ids":["https://openalex.org/I154425047"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058065354","display_name":"Owen Randall","orcid":null},"institutions":[{"id":"https://openalex.org/I154425047","display_name":"University of Alberta","ror":"https://ror.org/0160cpw27","country_code":"CA","type":"education","lineage":["https://openalex.org/I154425047"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Owen Randall","raw_affiliation_strings":["University of Alberta"],"affiliations":[{"raw_affiliation_string":"University of Alberta","institution_ids":["https://openalex.org/I154425047"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111899923","display_name":"Paul Lu","orcid":"https://orcid.org/0000-0002-4145-8056"},"institutions":[{"id":"https://openalex.org/I154425047","display_name":"University of Alberta","ror":"https://ror.org/0160cpw27","country_code":"CA","type":"education","lineage":["https://openalex.org/I154425047"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Paul Lu","raw_affiliation_strings":["University of Alberta"],"affiliations":[{"raw_affiliation_string":"University of Alberta","institution_ids":["https://openalex.org/I154425047"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I154425047"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.29508222,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"312","last_page":"321"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11614","display_name":"Cloud Data Security Solutions","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-deduplication","display_name":"Data deduplication","score":0.9322019219398499},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7207139134407043},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.40327972173690796},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.21612167358398438}],"concepts":[{"id":"https://openalex.org/C32587265","wikidata":"https://www.wikidata.org/wiki/Q1182260","display_name":"Data deduplication","level":2,"score":0.9322019219398499},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7207139134407043},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40327972173690796},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.21612167358398438}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata62323.2024.10825314","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10825314","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/13","score":0.4399999976158142,"display_name":"Climate action"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":32,"referenced_works":["https://openalex.org/W1505654100","https://openalex.org/W1614703486","https://openalex.org/W1969335064","https://openalex.org/W2005611725","https://openalex.org/W2038908732","https://openalex.org/W2096670209","https://openalex.org/W2107551255","https://openalex.org/W2110322986","https://openalex.org/W2477823473","https://openalex.org/W2734941459","https://openalex.org/W2799944323","https://openalex.org/W2912122742","https://openalex.org/W2912574597","https://openalex.org/W2914062858","https://openalex.org/W2998583855","https://openalex.org/W3008714254","https://openalex.org/W4236944670","https://openalex.org/W4318148204","https://openalex.org/W4391508056","https://openalex.org/W6600884019","https://openalex.org/W6601060166","https://openalex.org/W6602837329","https://openalex.org/W6607150254","https://openalex.org/W6608138927","https://openalex.org/W6628236356","https://openalex.org/W6628275343","https://openalex.org/W6631269574","https://openalex.org/W6640146480","https://openalex.org/W6679252032","https://openalex.org/W6705331335","https://openalex.org/W6776225533","https://openalex.org/W6793350840"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W3144870715","https://openalex.org/W3142319788","https://openalex.org/W2587188779","https://openalex.org/W3132870970","https://openalex.org/W4385804830","https://openalex.org/W2943088381","https://openalex.org/W2074021203"],"abstract_inverted_index":{"Content-defined":[0],"chunking":[1],"(CDC)":[2],"based":[3],"data":[4],"deduplication":[5,33,56,64,113],"is":[6,89,213],"a":[7,206,214],"complex":[8],"process,":[9,57],"leading":[10],"to":[11,31,40,61,168,183,201,223],"the":[12,51,54,70,79,105,164,193],"use":[13],"of":[14,53,74,136,209],"rule-of-thumb":[15,207],"approaches":[16,27],"and":[17,35,58,133,145,204,226],"standardized":[18],"parameter":[19,102,166,197],"values.":[20,155],"However,":[21,186],"our":[22,190,219],"work":[23],"challenges":[24],"these":[25,159],"standard":[26,134],"which":[28],"can":[29,110,121,150,161],"lead":[30],"worse":[32],"ratios,":[34],"reemphasizes":[36],"that":[37,85,118,129,158,192,205],"parameters":[38,179],"need":[39,200],"be":[41,122,181,202],"optimized":[42],"for":[43,91,142],"each":[44,184],"dataset.":[45,185],"We":[46,115,156,217],"expose":[47],"new":[48],"pitfalls,":[49],"analyze":[50],"behaviour":[52],"underlying":[55],"provide":[59,218],"solutions":[60],"aid":[62],"future":[63],"work.Deduplication":[65],"research":[66],"often":[67],"solely":[68],"reports":[69],"expected":[71,86,107,131],"chunk":[72,87,108,125,137,171,195],"length":[73,88,109,126,138,172,196],"their":[75],"system":[76],"without":[77],"providing":[78],"low-level":[80],"parameters.":[81],"Our":[82],"results":[83],"show":[84,117,157],"inadequate":[90],"properly":[92],"describing":[93],"deduplication,":[94],"making":[95],"empirical":[96],"reproducibility":[97,225],"challenging.":[98],"In":[99],"fact,":[100],"different":[101,112,170,175],"sets":[103],"with":[104],"same":[106,165],"yield":[111],"ratios.":[114],"further":[116,227],"this":[119],"discrepancy":[120],"explained":[123],"by":[124],"variance.We":[127],"find":[128,188],"because":[130],"average":[132],"deviation":[135],"do":[139],"not":[140,199],"account":[141],"file":[143],"boundaries":[144],"fingerprint":[146],"value":[147,208],"noise,":[148],"they":[149],"significantly":[151],"differ":[152],"from":[153],"observed":[154],"phenomena":[160],"also":[162],"cause":[163],"set":[167],"give":[169],"distributions":[173],"on":[174,189],"datasets,":[176],"explaining":[177],"why":[178],"must":[180],"tuned":[182],"we":[187],"datasets":[191,220],"maximum":[194],"does":[198],"tuned,":[203],"2<sup":[210],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[211],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">16</sup>":[212],"reasonable":[215],"selection.":[216],"in":[221],"full":[222],"promote":[224],"work.":[228]},"counts_by_year":[],"updated_date":"2026-04-17T18:11:37.981687","created_date":"2025-10-10T00:00:00"}