{"id":"https://openalex.org/W2270436279","doi":"https://doi.org/10.1093/comjnl/bxv052","title":"Large-Scale Schema-Free Data Deduplication Approach with Adaptive Sliding Window Using MapReduce","display_name":"Large-Scale Schema-Free Data Deduplication Approach with Adaptive Sliding Window Using MapReduce","publication_year":2015,"publication_date":"2015-07-18","ids":{"openalex":"https://openalex.org/W2270436279","doi":"https://doi.org/10.1093/comjnl/bxv052","mag":"2270436279"},"language":"en","primary_location":{"id":"doi:10.1093/comjnl/bxv052","is_oa":false,"landing_page_url":"https://doi.org/10.1093/comjnl/bxv052","pdf_url":null,"source":{"id":"https://openalex.org/S44643521","display_name":"The Computer Journal","issn_l":"0010-4620","issn":["0010-4620","1460-2067"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Computer Journal","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039282629","display_name":"Kun Ma","orcid":"https://orcid.org/0000-0002-0135-5423"},"institutions":[{"id":"https://openalex.org/I34949971","display_name":"University of Jinan","ror":"https://ror.org/02mjz6f26","country_code":"CN","type":"education","lineage":["https://openalex.org/I34949971"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Kun Ma","raw_affiliation_strings":["Shandong Provincial Key Laboratory of Network Based Intelligent Computing, University of Jinan, Jinan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shandong Provincial Key Laboratory of Network Based Intelligent Computing, University of Jinan, Jinan, China","institution_ids":["https://openalex.org/I34949971"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038192769","display_name":"Fusen Dong","orcid":null},"institutions":[{"id":"https://openalex.org/I34949971","display_name":"University of Jinan","ror":"https://ror.org/02mjz6f26","country_code":"CN","type":"education","lineage":["https://openalex.org/I34949971"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fusen Dong","raw_affiliation_strings":["Shandong Provincial Key Laboratory of Network Based Intelligent Computing, University of Jinan, Jinan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shandong Provincial Key Laboratory of Network Based Intelligent Computing, University of Jinan, Jinan, China","institution_ids":["https://openalex.org/I34949971"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5065368480","display_name":"Bo Yang","orcid":"https://orcid.org/0000-0001-7961-853X"},"institutions":[{"id":"https://openalex.org/I34949971","display_name":"University of Jinan","ror":"https://ror.org/02mjz6f26","country_code":"CN","type":"education","lineage":["https://openalex.org/I34949971"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bo Yang","raw_affiliation_strings":["Shandong Provincial Key Laboratory of Network Based Intelligent Computing, University of Jinan, Jinan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shandong Provincial Key Laboratory of Network Based Intelligent Computing, University of Jinan, Jinan, China","institution_ids":["https://openalex.org/I34949971"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5039282629"],"corresponding_institution_ids":["https://openalex.org/I34949971"],"apc_list":{"value":2635,"currency":"GBP","value_usd":3232},"apc_paid":null,"fwci":4.0077,"has_fulltext":false,"cited_by_count":24,"citation_normalized_percentile":{"value":0.93799018,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"58","issue":"11","first_page":"3187","last_page":"3201"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10538","display_name":"Data Mining Algorithms and Applications","score":0.9879000186920166,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11614","display_name":"Cloud Data Security Solutions","score":0.982699990272522,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-deduplication","display_name":"Data deduplication","score":0.9853746294975281},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8823604583740234},{"id":"https://openalex.org/keywords/sliding-window-protocol","display_name":"Sliding window protocol","score":0.731293261051178},{"id":"https://openalex.org/keywords/schema","display_name":"Schema (genetic algorithms)","score":0.5332932472229004},{"id":"https://openalex.org/keywords/window","display_name":"Window (computing)","score":0.4694159924983978},{"id":"https://openalex.org/keywords/partition","display_name":"Partition (number theory)","score":0.4443245828151703},{"id":"https://openalex.org/keywords/block","display_name":"Block (permutation group theory)","score":0.44048312306404114},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4307040572166443},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.21098753809928894},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.08489242196083069}],"concepts":[{"id":"https://openalex.org/C32587265","wikidata":"https://www.wikidata.org/wiki/Q1182260","display_name":"Data deduplication","level":2,"score":0.9853746294975281},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8823604583740234},{"id":"https://openalex.org/C102392041","wikidata":"https://www.wikidata.org/wiki/Q592860","display_name":"Sliding window protocol","level":3,"score":0.731293261051178},{"id":"https://openalex.org/C52146309","wikidata":"https://www.wikidata.org/wiki/Q7431116","display_name":"Schema (genetic algorithms)","level":2,"score":0.5332932472229004},{"id":"https://openalex.org/C2778751112","wikidata":"https://www.wikidata.org/wiki/Q835016","display_name":"Window (computing)","level":2,"score":0.4694159924983978},{"id":"https://openalex.org/C42812","wikidata":"https://www.wikidata.org/wiki/Q1082910","display_name":"Partition (number theory)","level":2,"score":0.4443245828151703},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.44048312306404114},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4307040572166443},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.21098753809928894},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.08489242196083069},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1093/comjnl/bxv052","is_oa":false,"landing_page_url":"https://doi.org/10.1093/comjnl/bxv052","pdf_url":null,"source":{"id":"https://openalex.org/S44643521","display_name":"The Computer Journal","issn_l":"0010-4620","issn":["0010-4620","1460-2067"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Computer Journal","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/2","display_name":"Zero hunger","score":0.6200000047683716}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W26430610","https://openalex.org/W87992039","https://openalex.org/W118209581","https://openalex.org/W154681553","https://openalex.org/W1903539383","https://openalex.org/W1981590391","https://openalex.org/W1985081769","https://openalex.org/W1986017822","https://openalex.org/W2020333861","https://openalex.org/W2024404542","https://openalex.org/W2024770506","https://openalex.org/W2027393969","https://openalex.org/W2029084939","https://openalex.org/W2031250218","https://openalex.org/W2039001182","https://openalex.org/W2041439319","https://openalex.org/W2045202139","https://openalex.org/W2046626049","https://openalex.org/W2056748234","https://openalex.org/W2058733945","https://openalex.org/W2073471108","https://openalex.org/W2074021203","https://openalex.org/W2077044633","https://openalex.org/W2077673618","https://openalex.org/W2079971532","https://openalex.org/W2084568010","https://openalex.org/W2097225630","https://openalex.org/W2104511295","https://openalex.org/W2106521941","https://openalex.org/W2108991785","https://openalex.org/W2111116800","https://openalex.org/W2114296561","https://openalex.org/W2117974736","https://openalex.org/W2129319305","https://openalex.org/W2130846267","https://openalex.org/W2140688797","https://openalex.org/W2150228342","https://openalex.org/W2151930506","https://openalex.org/W2154785834","https://openalex.org/W2157355837","https://openalex.org/W2160484851","https://openalex.org/W2163847742","https://openalex.org/W2165769837","https://openalex.org/W2166988329","https://openalex.org/W2169024178","https://openalex.org/W2173213060","https://openalex.org/W2185002504","https://openalex.org/W2200896674","https://openalex.org/W2209768828","https://openalex.org/W2223011434","https://openalex.org/W2295151155","https://openalex.org/W2320323244","https://openalex.org/W3146259567","https://openalex.org/W4230502578","https://openalex.org/W4230511558","https://openalex.org/W4254788633","https://openalex.org/W6686276466","https://openalex.org/W6700621067"],"related_works":["https://openalex.org/W2141008330","https://openalex.org/W6445124","https://openalex.org/W2915462855","https://openalex.org/W2365320428","https://openalex.org/W1999414040","https://openalex.org/W2092682806","https://openalex.org/W3005328281","https://openalex.org/W2991818765","https://openalex.org/W2363017932","https://openalex.org/W2327832173"],"abstract_inverted_index":{"Data":[0],"deduplication":[1,40,50,74,92,137],"is":[2],"the":[3,25,45,136,145,152,158],"task":[4,20],"of":[5,9,27,47,62,157],"identifying":[6],"all":[7],"groups":[8],"objects":[10,118],"within":[11],"one":[12],"or":[13],"several":[14],"data":[15,39,49,147],"sets,":[16],"respectively.":[17],"However,":[18],"this":[19,32],"will":[21],"become":[22],"difficult":[23],"in":[24,42,44,69,119],"context":[26],"big":[28],"data.":[29],"To":[30],"address":[31],"limitation,":[33],"we":[34,85,123],"propose":[35,124],"a":[36,125],"new":[37],"schema-free":[38],"approach":[41,128],"parallel":[43,60],"aspect":[46],"breeding":[48,146],"related":[51],"to":[52,80,89,96,133],"food":[53],"safety.":[54],"Although":[55],"MapReduce":[56,77],"framework":[57],"enables":[58],"efficient":[59],"execution":[61],"data-intensive":[63],"tasks,":[64],"it":[65],"cannot":[66],"find":[67,113],"duplicates":[68,114],"adjacent":[70,120],"block.":[71],"Furthermore,":[72],"current":[73,91],"approaches":[75,93],"with":[76,94,108,129],"are":[78],"restricted":[79],"fixed":[81],"sliding":[82,98,131],"window.":[83],"Therefore,":[84],"investigate":[86],"possible":[87],"solutions":[88],"improve":[90],"MapReduce,":[95],"make":[97],"window":[99,110,132],"size":[100],"adaptive":[101,103,130],"using":[102],"multiple":[104],"duplicate":[105],"count":[106],"strategy":[107],"alterable":[109],"step,":[111],"and":[112,155],"by":[115],"overlapping":[116],"boundary":[117],"blocks.":[121],"Moreover,":[122],"multi-pass":[126],"Partition-Sort-Map-Reduce":[127],"speed":[134],"up":[135],"process.":[138],"Finally,":[139],"our":[140],"experimental":[141],"evaluation":[142],"based":[143],"on":[144,148],"large":[149],"datasets":[150],"shows":[151],"high":[153],"effectiveness":[154],"efficiency":[156],"proposed":[159],"approaches.":[160]},"counts_by_year":[{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}