{"id":"https://openalex.org/W4230077428","doi":"https://doi.org/10.1109/sc.2012.14","title":"A study on data deduplication in HPC storage systems","display_name":"A study on data deduplication in HPC storage systems","publication_year":2012,"publication_date":"2012-11-01","ids":{"openalex":"https://openalex.org/W4230077428","doi":"https://doi.org/10.1109/sc.2012.14"},"language":"en","primary_location":{"id":"doi:10.1109/sc.2012.14","is_oa":false,"landing_page_url":"https://doi.org/10.1109/sc.2012.14","pdf_url":null,"source":{"id":"https://openalex.org/S4363608557","display_name":"2012 International Conference for High Performance Computing, Networking, Storage and Analysis","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2012 International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003807012","display_name":"Dirk Meister","orcid":null},"institutions":[{"id":"https://openalex.org/I197323543","display_name":"Johannes Gutenberg University Mainz","ror":"https://ror.org/023b0x485","country_code":"DE","type":"education","lineage":["https://openalex.org/I197323543"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Dirk Meister","raw_affiliation_strings":["Johannes Gutenberg-University, Mainz, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johannes Gutenberg-University, Mainz, Germany","institution_ids":["https://openalex.org/I197323543"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049694671","display_name":"J\u00fcrgen Kaiser","orcid":null},"institutions":[{"id":"https://openalex.org/I197323543","display_name":"Johannes Gutenberg University Mainz","ror":"https://ror.org/023b0x485","country_code":"DE","type":"education","lineage":["https://openalex.org/I197323543"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Jurgen Kaiser","raw_affiliation_strings":["Johannes Gutenberg-University, Mainz, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johannes Gutenberg-University, Mainz, Germany","institution_ids":["https://openalex.org/I197323543"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011466225","display_name":"Andr\u00e9 Brinkmann","orcid":"https://orcid.org/0000-0003-3083-2775"},"institutions":[{"id":"https://openalex.org/I197323543","display_name":"Johannes Gutenberg University Mainz","ror":"https://ror.org/023b0x485","country_code":"DE","type":"education","lineage":["https://openalex.org/I197323543"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Andre Brinkmann","raw_affiliation_strings":["Johannes Gutenberg-University, Mainz, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johannes Gutenberg-University, Mainz, Germany","institution_ids":["https://openalex.org/I197323543"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036494255","display_name":"Toni Cort\u00e9s","orcid":"https://orcid.org/0000-0002-2537-8937"},"institutions":[{"id":"https://openalex.org/I2799803557","display_name":"Barcelona Supercomputing Center","ror":"https://ror.org/05sd8tv96","country_code":"ES","type":"facility","lineage":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]},{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Toni Cortes","raw_affiliation_strings":["Barcelona Supercomputing Center, Universitat Polit\u00e8cnica de Catalunya, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Barcelona Supercomputing Center, Universitat Polit\u00e8cnica de Catalunya, Spain","institution_ids":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028324537","display_name":"Michael K\u00fchn","orcid":"https://orcid.org/0000-0001-8167-8574"},"institutions":[{"id":"https://openalex.org/I159176309","display_name":"Universit\u00e4t Hamburg","ror":"https://ror.org/00g30e956","country_code":"DE","type":"education","lineage":["https://openalex.org/I159176309"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Michael Kuhn","raw_affiliation_strings":["University of Hamburg, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Hamburg, Germany","institution_ids":["https://openalex.org/I159176309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028731194","display_name":"Julian Kunkel","orcid":"https://orcid.org/0000-0002-6915-1179"},"institutions":[{"id":"https://openalex.org/I159176309","display_name":"Universit\u00e4t Hamburg","ror":"https://ror.org/00g30e956","country_code":"DE","type":"education","lineage":["https://openalex.org/I159176309"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Julian Kunkel","raw_affiliation_strings":["University of Hamburg, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Hamburg, Germany","institution_ids":["https://openalex.org/I159176309"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":95,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11614","display_name":"Cloud Data Security Solutions","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9954000115394592,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-deduplication","display_name":"Data deduplication","score":0.9773157835006714},{"id":"https://openalex.org/keywords/backup","display_name":"Backup","score":0.8873182535171509},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8444148898124695},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.6959403157234192},{"id":"https://openalex.org/keywords/data-redundancy","display_name":"Data redundancy","score":0.5317254066467285},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.497592955827713},{"id":"https://openalex.org/keywords/computer-data-storage","display_name":"Computer data storage","score":0.4848805367946625},{"id":"https://openalex.org/keywords/block","display_name":"Block (permutation group theory)","score":0.4345186948776245},{"id":"https://openalex.org/keywords/file-system","display_name":"File system","score":0.41521814465522766},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.2534372806549072}],"concepts":[{"id":"https://openalex.org/C32587265","wikidata":"https://www.wikidata.org/wiki/Q1182260","display_name":"Data deduplication","level":2,"score":0.9773157835006714},{"id":"https://openalex.org/C2780945871","wikidata":"https://www.wikidata.org/wiki/Q194274","display_name":"Backup","level":2,"score":0.8873182535171509},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8444148898124695},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.6959403157234192},{"id":"https://openalex.org/C7545210","wikidata":"https://www.wikidata.org/wiki/Q838123","display_name":"Data redundancy","level":2,"score":0.5317254066467285},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.497592955827713},{"id":"https://openalex.org/C194739806","wikidata":"https://www.wikidata.org/wiki/Q66221","display_name":"Computer data storage","level":2,"score":0.4848805367946625},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.4345186948776245},{"id":"https://openalex.org/C2780940931","wikidata":"https://www.wikidata.org/wiki/Q174989","display_name":"File system","level":2,"score":0.41521814465522766},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2534372806549072},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/sc.2012.14","is_oa":false,"landing_page_url":"https://doi.org/10.1109/sc.2012.14","pdf_url":null,"source":{"id":"https://openalex.org/S4363608557","display_name":"2012 International Conference for High Performance Computing, Networking, Storage and Analysis","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2012 International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"},{"id":"pmh:oai:upcommons.upc.edu:2117/20270","is_oa":false,"landing_page_url":"http://hdl.handle.net/2117/20270","pdf_url":null,"source":{"id":"https://openalex.org/S4377196262","display_name":"UPCommons institutional repository (Universitat Polit\u00e8cnica de Catalunya)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I9617848","host_organization_name":"Universitat Polit\u00e8cnica de Catalunya","host_organization_lineage":["https://openalex.org/I9617848"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference report"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321469","display_name":"Bundesministerium f\u00fcr Wirtschaft und Technologie","ror":"https://ror.org/02vgg2808"},{"id":"https://openalex.org/F4320321837","display_name":"Ministerio de Econom\u00eda y Competitividad","ror":"https://ror.org/034900433"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":56,"referenced_works":["https://openalex.org/W22807665","https://openalex.org/W24740120","https://openalex.org/W66778775","https://openalex.org/W146191923","https://openalex.org/W174289249","https://openalex.org/W193648907","https://openalex.org/W200233886","https://openalex.org/W1508926177","https://openalex.org/W1576397915","https://openalex.org/W1969126835","https://openalex.org/W1971212200","https://openalex.org/W1998817683","https://openalex.org/W1999984505","https://openalex.org/W2003131282","https://openalex.org/W2003696547","https://openalex.org/W2035415465","https://openalex.org/W2038908732","https://openalex.org/W2047484077","https://openalex.org/W2056980397","https://openalex.org/W2065077642","https://openalex.org/W2070834743","https://openalex.org/W2071789190","https://openalex.org/W2073370301","https://openalex.org/W2089536264","https://openalex.org/W2098935637","https://openalex.org/W2107745473","https://openalex.org/W2110298485","https://openalex.org/W2110322986","https://openalex.org/W2128593562","https://openalex.org/W2130975798","https://openalex.org/W2131645490","https://openalex.org/W2134792144","https://openalex.org/W2144554431","https://openalex.org/W2153793343","https://openalex.org/W2160127232","https://openalex.org/W2162090341","https://openalex.org/W2167208373","https://openalex.org/W2171796885","https://openalex.org/W2254416046","https://openalex.org/W4236236547","https://openalex.org/W4289127653","https://openalex.org/W6600884019","https://openalex.org/W6601060166","https://openalex.org/W6602654064","https://openalex.org/W6605588685","https://openalex.org/W6607057542","https://openalex.org/W6607794886","https://openalex.org/W6608138927","https://openalex.org/W6630499100","https://openalex.org/W6634463810","https://openalex.org/W6650418438","https://openalex.org/W6676560735","https://openalex.org/W6679252032","https://openalex.org/W6679471073","https://openalex.org/W6691979229","https://openalex.org/W6845441823"],"related_works":["https://openalex.org/W4285245558","https://openalex.org/W1437397736","https://openalex.org/W2048606985","https://openalex.org/W2375056709","https://openalex.org/W4301155776","https://openalex.org/W2012359782","https://openalex.org/W2952411620","https://openalex.org/W2348132657","https://openalex.org/W3008013744","https://openalex.org/W2077135008"],"abstract_inverted_index":{"Deduplication":[0],"is":[1,7,48,52,100],"a":[2,15,18,35,83,98,210],"storage":[3,132],"saving":[4],"technique":[5],"that":[6,39,62,178],"highly":[8],"successful":[9],"in":[10,123],"enterprise":[11],"backup":[12,160],"environments.":[13],"On":[14],"file":[16,36,77,172],"system,":[17],"single":[19],"data":[20,46,57,78,121,145,186,192,201],"block":[21],"might":[22,37],"be":[23,188,207],"stored":[24],"multiple":[25,32],"times":[26],"across":[27],"different":[28],"files,":[29],"for":[30,86,140,143,199],"example,":[31],"versions":[33],"of":[34,120,170,183],"exist":[38],"are":[40,109],"mostly":[41,158],"identical.":[42],"With":[43],"deduplication,":[44],"this":[45,138,184],"replication":[47],"localized":[49],"and":[50,81],"redundancy":[51],"removed":[53,189],"--":[54],"by":[55,190,209],"storing":[56],"just":[58],"once,":[59],"all":[60],"files":[61],"use":[63],"identical":[64],"regions":[65],"refer":[66],"to":[67,128,153,181,197,222],"the":[68,92,114,118,129],"same":[69],"unique":[70,107],"data.":[71,174],"The":[72,175],"most":[73,130],"common":[74],"approach":[75],"splits":[76],"into":[79],"chunks":[80,108],"calculates":[82],"cryptographic":[84],"fingerprint":[85,93],"each":[87],"chunk.":[88],"By":[89],"checking":[90],"if":[91],"has":[94],"already":[95],"been":[96],"stored,":[97],"chunk":[99],"classified":[101],"as":[102],"redundant":[103],"or":[104],"unique.":[105],"Only":[106],"stored.":[110],"This":[111,203],"paper":[112],"presents":[113],"first":[115],"study":[116],"on":[117,159,217],"potential":[119,139],"deduplication":[122,155,193,212],"HPC":[124],"centers,":[125],"which":[126],"belong":[127],"demanding":[131],"producers.":[133],"We":[134],"have":[135,163],"quantitatively":[136],"assessed":[137],"capacity":[141,224],"reduction":[142,204],"4":[144],"centers":[146],"(BSC,":[147],"DKRZ,":[148],"RENCI,":[149],"RWTH).":[150],"In":[151],"contrast":[152],"previous":[154],"studies":[156],"focusing":[157],"data,":[161],"we":[162],"analyzed":[164],"over":[165],"one":[166],"PB":[167],"(1212":[168],"TB)":[169],"online":[171,185],"system":[173],"evaluation":[176],"shows":[177],"typically":[179],"20%":[180],"30%":[182],"can":[187,205],"applying":[191],"techniques,":[194],"peaking":[195],"up":[196],"70%":[198],"some":[200],"sets.":[202],"only":[206,220],"achieved":[208],"subfile":[211],"approach,":[213],"while":[214],"approaches":[215],"based":[216],"whole-file":[218],"comparisons":[219],"lead":[221],"small":[223],"savings.":[225]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":11},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":10},{"year":2018,"cited_by_count":14},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":18},{"year":2015,"cited_by_count":5},{"year":2014,"cited_by_count":4},{"year":2013,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}