{"id":"https://openalex.org/W1992768421","doi":"https://doi.org/10.1155/2014/561340","title":"Design and Implementation of File Deduplication Framework on HDFS","display_name":"Design and Implementation of File Deduplication Framework on HDFS","publication_year":2014,"publication_date":"2014-04-01","ids":{"openalex":"https://openalex.org/W1992768421","doi":"https://doi.org/10.1155/2014/561340","mag":"1992768421"},"language":"en","primary_location":{"id":"doi:10.1155/2014/561340","is_oa":true,"landing_page_url":"https://doi.org/10.1155/2014/561340","pdf_url":null,"source":{"id":"https://openalex.org/S64417657","display_name":"International Journal of Distributed Sensor Networks","issn_l":"1550-1329","issn":["1550-1329","1550-1477"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319869","host_organization_name":"Hindawi Publishing Corporation","host_organization_lineage":["https://openalex.org/P4310319869","https://openalex.org/P4310320595"],"host_organization_lineage_names":["Hindawi Publishing Corporation","Wiley"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Distributed Sensor Networks","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1155/2014/561340","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015103550","display_name":"Ruey\u2010Kai Sheu","orcid":"https://orcid.org/0000-0002-3014-8095"},"institutions":[{"id":"https://openalex.org/I169090423","display_name":"Tunghai University","ror":"https://ror.org/00zhvdn11","country_code":"TW","type":"education","lineage":["https://openalex.org/I169090423"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Ruey-Kai Sheu","raw_affiliation_strings":["Department of Computer Science, Tung Hai University, Taichung, Taiwan","1 Department of Computer Science, Tung Hai University, Taichung, Taiwan"],"raw_orcid":"https://orcid.org/0000-0002-3014-8095","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Tung Hai University, Taichung, Taiwan","institution_ids":["https://openalex.org/I169090423"]},{"raw_affiliation_string":"1 Department of Computer Science, Tung Hai University, Taichung, Taiwan","institution_ids":["https://openalex.org/I169090423"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100629585","display_name":"Shyan\u2010Ming Yuan","orcid":"https://orcid.org/0000-0002-3621-9528"},"institutions":[{"id":"https://openalex.org/I148366613","display_name":"National Yang Ming Chiao Tung University","ror":"https://ror.org/00se2k293","country_code":"TW","type":"education","lineage":["https://openalex.org/I148366613"]}],"countries":["TW"],"is_corresponding":true,"raw_author_name":"Shyan-Ming Yuan","raw_affiliation_strings":["Institute of Computer Science and Information Engineering, National Chiao Tung University, Hsinchu, Taiwan","2 Institute of Computer Science and Information Engineering, National Chiao Tung University, Hsinchu, Taiwan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computer Science and Information Engineering, National Chiao Tung University, Hsinchu, Taiwan","institution_ids":["https://openalex.org/I148366613"]},{"raw_affiliation_string":"2 Institute of Computer Science and Information Engineering, National Chiao Tung University, Hsinchu, Taiwan","institution_ids":["https://openalex.org/I148366613"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016233136","display_name":"Win\u2010Tsung Lo","orcid":"https://orcid.org/0000-0002-9306-5713"},"institutions":[{"id":"https://openalex.org/I169090423","display_name":"Tunghai University","ror":"https://ror.org/00zhvdn11","country_code":"TW","type":"education","lineage":["https://openalex.org/I169090423"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Win-Tsung Lo","raw_affiliation_strings":["Department of Computer Science, Tung Hai University, Taichung, Taiwan","1 Department of Computer Science, Tung Hai University, Taichung, Taiwan"],"raw_orcid":"https://orcid.org/0000-0002-9306-5713","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Tung Hai University, Taichung, Taiwan","institution_ids":["https://openalex.org/I169090423"]},{"raw_affiliation_string":"1 Department of Computer Science, Tung Hai University, Taichung, Taiwan","institution_ids":["https://openalex.org/I169090423"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077106779","display_name":"Chan-I Ku","orcid":null},"institutions":[{"id":"https://openalex.org/I142066694","display_name":"ITRI International","ror":"https://ror.org/04wwsbd59","country_code":"US","type":"facility","lineage":["https://openalex.org/I142066694"]},{"id":"https://openalex.org/I4210148468","display_name":"Industrial Technology Research Institute","ror":"https://ror.org/05szzwt63","country_code":"TW","type":"nonprofit","lineage":["https://openalex.org/I4210148468"]}],"countries":["TW","US"],"is_corresponding":false,"raw_author_name":"Chan-I Ku","raw_affiliation_strings":["Computational Intelligence Technology Center, Industrial Technology Research Institute, Hsinchu, Taiwan","3 Computational Intelligence Technology Center, Industrial Technology Research Institute, Hsinchu, Taiwan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computational Intelligence Technology Center, Industrial Technology Research Institute, Hsinchu, Taiwan","institution_ids":["https://openalex.org/I4210148468"]},{"raw_affiliation_string":"3 Computational Intelligence Technology Center, Industrial Technology Research Institute, Hsinchu, Taiwan","institution_ids":["https://openalex.org/I142066694"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5100629585"],"corresponding_institution_ids":["https://openalex.org/I148366613"],"apc_list":{"value":2200,"currency":"USD","value_usd":2200},"apc_paid":{"value":2200,"currency":"USD","value_usd":2200},"fwci":7.7311,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.96912493,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"10","issue":"4","first_page":"561340","last_page":"561340"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11614","display_name":"Cloud Data Security Solutions","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11614","display_name":"Cloud Data Security Solutions","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9894000291824341,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9873999953269958,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8998191356658936},{"id":"https://openalex.org/keywords/data-deduplication","display_name":"Data deduplication","score":0.7759171724319458},{"id":"https://openalex.org/keywords/unix-file-types","display_name":"Unix file types","score":0.7290737628936768},{"id":"https://openalex.org/keywords/self-certifying-file-system","display_name":"Self-certifying File System","score":0.6962871551513672},{"id":"https://openalex.org/keywords/file-system","display_name":"File system","score":0.6654202938079834},{"id":"https://openalex.org/keywords/file-system-fragmentation","display_name":"File system fragmentation","score":0.6597524881362915},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.6522699594497681},{"id":"https://openalex.org/keywords/backup","display_name":"Backup","score":0.6490893363952637},{"id":"https://openalex.org/keywords/versioning-file-system","display_name":"Versioning file system","score":0.6115564107894897},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.5717208385467529},{"id":"https://openalex.org/keywords/torrent-file","display_name":"Torrent file","score":0.5661234855651855},{"id":"https://openalex.org/keywords/distributed-file-system","display_name":"Distributed File System","score":0.5552920699119568},{"id":"https://openalex.org/keywords/computer-file","display_name":"Computer file","score":0.5205134749412537},{"id":"https://openalex.org/keywords/virtual-file-system","display_name":"Virtual file system","score":0.49875760078430176},{"id":"https://openalex.org/keywords/cloud-storage","display_name":"Cloud storage","score":0.46151089668273926},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.447967529296875},{"id":"https://openalex.org/keywords/file-control-block","display_name":"File Control Block","score":0.4424409866333008},{"id":"https://openalex.org/keywords/server","display_name":"Server","score":0.4224206507205963},{"id":"https://openalex.org/keywords/ssh-file-transfer-protocol","display_name":"SSH File Transfer Protocol","score":0.41859710216522217},{"id":"https://openalex.org/keywords/device-file","display_name":"Device file","score":0.38179653882980347},{"id":"https://openalex.org/keywords/stub-file","display_name":"Stub file","score":0.34813809394836426}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8998191356658936},{"id":"https://openalex.org/C32587265","wikidata":"https://www.wikidata.org/wiki/Q1182260","display_name":"Data deduplication","level":2,"score":0.7759171724319458},{"id":"https://openalex.org/C21729314","wikidata":"https://www.wikidata.org/wiki/Q7896858","display_name":"Unix file types","level":4,"score":0.7290737628936768},{"id":"https://openalex.org/C82820731","wikidata":"https://www.wikidata.org/wiki/Q2640620","display_name":"Self-certifying File System","level":3,"score":0.6962871551513672},{"id":"https://openalex.org/C2780940931","wikidata":"https://www.wikidata.org/wiki/Q174989","display_name":"File system","level":2,"score":0.6654202938079834},{"id":"https://openalex.org/C26656859","wikidata":"https://www.wikidata.org/wiki/Q4089244","display_name":"File system fragmentation","level":4,"score":0.6597524881362915},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.6522699594497681},{"id":"https://openalex.org/C2780945871","wikidata":"https://www.wikidata.org/wiki/Q194274","display_name":"Backup","level":2,"score":0.6490893363952637},{"id":"https://openalex.org/C166807848","wikidata":"https://www.wikidata.org/wiki/Q3445065","display_name":"Versioning file system","level":4,"score":0.6115564107894897},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.5717208385467529},{"id":"https://openalex.org/C40350719","wikidata":"https://www.wikidata.org/wiki/Q2451637","display_name":"Torrent file","level":4,"score":0.5661234855651855},{"id":"https://openalex.org/C152043487","wikidata":"https://www.wikidata.org/wiki/Q1229600","display_name":"Distributed File System","level":2,"score":0.5552920699119568},{"id":"https://openalex.org/C95637964","wikidata":"https://www.wikidata.org/wiki/Q82753","display_name":"Computer file","level":2,"score":0.5205134749412537},{"id":"https://openalex.org/C58861120","wikidata":"https://www.wikidata.org/wiki/Q1330734","display_name":"Virtual file system","level":5,"score":0.49875760078430176},{"id":"https://openalex.org/C2777059624","wikidata":"https://www.wikidata.org/wiki/Q914359","display_name":"Cloud storage","level":3,"score":0.46151089668273926},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.447967529296875},{"id":"https://openalex.org/C180500224","wikidata":"https://www.wikidata.org/wiki/Q1412592","display_name":"File Control Block","level":4,"score":0.4424409866333008},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.4224206507205963},{"id":"https://openalex.org/C88520388","wikidata":"https://www.wikidata.org/wiki/Q1126823","display_name":"SSH File Transfer Protocol","level":4,"score":0.41859710216522217},{"id":"https://openalex.org/C193769178","wikidata":"https://www.wikidata.org/wiki/Q14679","display_name":"Device file","level":3,"score":0.38179653882980347},{"id":"https://openalex.org/C13674803","wikidata":"https://www.wikidata.org/wiki/Q7627301","display_name":"Stub file","level":3,"score":0.34813809394836426}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1155/2014/561340","is_oa":true,"landing_page_url":"https://doi.org/10.1155/2014/561340","pdf_url":null,"source":{"id":"https://openalex.org/S64417657","display_name":"International Journal of Distributed Sensor Networks","issn_l":"1550-1329","issn":["1550-1329","1550-1477"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319869","host_organization_name":"Hindawi Publishing Corporation","host_organization_lineage":["https://openalex.org/P4310319869","https://openalex.org/P4310320595"],"host_organization_lineage_names":["Hindawi Publishing Corporation","Wiley"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Distributed Sensor Networks","raw_type":"journal-article"},{"id":"pmh:oai:RePEc:sae:intdis:v:10:y:2014:i:4:p:561340","is_oa":false,"landing_page_url":"https://journals.sagepub.com/doi/10.1155/2014/561340","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"pmh:oai:doaj.org/article:724a93027ad74c0fa99e79dc6c0d01a6","is_oa":true,"landing_page_url":"https://doaj.org/article/724a93027ad74c0fa99e79dc6c0d01a6","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"International Journal of Distributed Sensor Networks, Vol 10 (2014)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1155/2014/561340","is_oa":true,"landing_page_url":"https://doi.org/10.1155/2014/561340","pdf_url":null,"source":{"id":"https://openalex.org/S64417657","display_name":"International Journal of Distributed Sensor Networks","issn_l":"1550-1329","issn":["1550-1329","1550-1477"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319869","host_organization_name":"Hindawi Publishing Corporation","host_organization_lineage":["https://openalex.org/P4310319869","https://openalex.org/P4310320595"],"host_organization_lineage_names":["Hindawi Publishing Corporation","Wiley"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Distributed Sensor Networks","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.6200000047683716,"display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G3248561460","display_name":null,"funder_award_id":"101-2221-E-009-034-MY2","funder_id":"https://openalex.org/F4320321040","funder_display_name":"National Science Council"}],"funders":[{"id":"https://openalex.org/F4320321040","display_name":"National Science Council","ror":"https://ror.org/02kv4zf79"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W6847055","https://openalex.org/W141925461","https://openalex.org/W1512773843","https://openalex.org/W1599186236","https://openalex.org/W1969067141","https://openalex.org/W1989135621","https://openalex.org/W1991578531","https://openalex.org/W1992527370","https://openalex.org/W2041799676","https://openalex.org/W2048377555","https://openalex.org/W2052207834","https://openalex.org/W2104511295","https://openalex.org/W2110322986","https://openalex.org/W2115075989","https://openalex.org/W2119565742","https://openalex.org/W2139731740","https://openalex.org/W2146889157","https://openalex.org/W2912574597"],"related_works":["https://openalex.org/W1990666357","https://openalex.org/W4247882433","https://openalex.org/W4313226861","https://openalex.org/W2098881120","https://openalex.org/W2342627122","https://openalex.org/W4396731530","https://openalex.org/W2140733653","https://openalex.org/W4250364891","https://openalex.org/W2023412278","https://openalex.org/W4233195927"],"abstract_inverted_index":{"File":[0],"systems":[1,22,75],"are":[2,8,86,143],"designed":[3],"to":[4,48,52,122],"control":[5],"how":[6],"files":[7],"stored":[9],"and":[10,16,27,35,43,55,91,140],"retrieved.":[11],"Without":[12],"knowing":[13],"the":[14,57,77,84,98,105,110,117],"context":[15],"semantics":[17],"of":[18,32,83,119],"file":[19,21,89,100,113,125,131],"contents,":[20],"often":[23],"contain":[24],"duplicate":[25],"copies":[26],"result":[28],"in":[29,146],"redundant":[30],"consumptions":[31],"storage":[33,58],"space":[34],"network":[36],"bandwidth.":[37],"It":[38,115],"has":[39],"been":[40],"a":[41,124],"complex":[42],"challenging":[44],"issue":[45],"for":[46,70,88,109,133],"enterprises":[47],"seek":[49],"deduplication":[50,102,126],"technologies":[51,85,103],"reduce":[53],"cost":[54],"increase":[56],"efficiency.":[59],"To":[60],"solve":[61],"such":[62],"problem,":[63],"researchers":[64],"proposed":[65],"in-line":[66],"or":[67,73,79],"offline":[68],"solutions":[69],"primary":[71],"storages":[72],"backup":[74],"at":[76,104],"subfile":[78],"whole-file":[80],"level.":[81],"Some":[82],"used":[87],"servers":[90],"database":[92],"systems.":[93],"Fewer":[94],"studies":[95],"focus":[96],"on":[97,128],"cloud":[99,134],"system":[101,132],"application":[106,135],"level,":[107],"especially":[108],"Hadoop":[111,129],"distributed":[112,130],"system.":[114],"is":[116],"goal":[118],"this":[120,147],"paper":[121],"design":[123],"framework":[127],"developers.":[136],"The":[137],"architecture,":[138],"interface,":[139],"implementation":[141],"experiences":[142],"also":[144],"shared":[145],"paper.":[148]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":2},{"year":2015,"cited_by_count":3}],"updated_date":"2026-08-15T07:11:24.734988","created_date":"2025-10-10T00:00:00"}