{"id":"https://openalex.org/W3000712052","doi":"https://doi.org/10.1109/bigdata47090.2019.9005703","title":"High Performance I/O For Large Scale Deep Learning","display_name":"High Performance I/O For Large Scale Deep Learning","alternate_titles":[],"publication_year":2019,"publication_date":"2019-12-01","ids":{"openalex":"https://openalex.org/W3000712052","doi":"https://doi.org/10.1109/bigdata47090.2019.9005703","mag":"3000712052"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata47090.2019.9005703","doi":"https://doi.org/10.1109/bigdata47090.2019.9005703","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata47090.2019.9005703","pdf_url":null,"source":{"id":"https://openalex.org/S4363607709","display_name":"IEEE International Conference on Big Data (BigData)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W3000712052"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2001.01858","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077575558","display_name":"Alex Aizman","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alex Aizman","raw_affiliation_strings":["NVIDIA, Santa Clara, CA, USA","nVidia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA, Santa Clara, CA, USA","institution_ids":["https://openalex.org/I4210127875"]},{"raw_affiliation_string":"nVidia","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077353309","display_name":"Gavin Maltby","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gavin Maltby","raw_affiliation_strings":["NVIDIA, Santa Clara, CA, USA","nVidia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA, Santa Clara, CA, USA","institution_ids":["https://openalex.org/I4210127875"]},{"raw_affiliation_string":"nVidia","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5088840119","display_name":"Thomas Michael Breuel","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Thomas Breuel","raw_affiliation_strings":["NVIDIA, Santa Clara, CA, USA","nVidia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA, Santa Clara, CA, USA","institution_ids":["https://openalex.org/I4210127875"]},{"raw_affiliation_string":"nVidia","institution_ids":["https://openalex.org/I4210127875"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210127875"],"apc_list":null,"apc_paid":null,"fwci":2.8089,"has_fulltext":true,"cited_by_count":66,"citation_normalized_percentile":{"value":0.92384888,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"5965","last_page":"5967"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.98580002784729,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.98580002784729,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.00989999994635582,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/distributed-storage","display_name":"distributed storage","score":0.6449999809265137},{"id":"https://openalex.org/keywords/deep-learning-training","display_name":"deep learning training","score":0.7570000290870667},{"id":"https://openalex.org/keywords/large-scale-datasets","display_name":"large-scale datasets","score":0.7329999804496765},{"id":"https://openalex.org/keywords/scalable-storage","display_name":"scalable storage","score":0.6179999709129333},{"id":"https://openalex.org/keywords/hadoop-distributed-file-system","display_name":"HDFS","score":0.7450000047683716},{"id":"https://openalex.org/keywords/nfs","display_name":"NFS","score":0.5429999828338623},{"id":"https://openalex.org/keywords/storage-backends","display_name":"storage backends","score":0.4659999907016754}],"concepts":[{"id":"https://openalex.org/C185410017","wikidata":"https://www.wikidata.org/wiki/Q7171778","display_name":"Petascale computing","level":3,"score":0.9228401184082031},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8577232956886292},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7921825647354126},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.6274729371070862},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.578120231628418},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.5437847375869751},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5318498015403748},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.47187915444374084},{"id":"https://openalex.org/C62611344","wikidata":"https://www.wikidata.org/wiki/Q1062658","display_name":"Node (physics)","level":2,"score":0.4592042565345764},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4335380792617798},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.430858850479126},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37731412053108215},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.3752891719341278},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3262370526790619},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2634287476539612},{"id":"https://openalex.org/C66938386","wikidata":"https://www.wikidata.org/wiki/Q633538","display_name":"Structural engineering","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/bigdata47090.2019.9005703","doi":"https://doi.org/10.1109/bigdata47090.2019.9005703","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata47090.2019.9005703","pdf_url":null,"source":{"id":"https://openalex.org/S4363607709","display_name":"IEEE International Conference on Big Data (BigData)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2001.01858","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2001.01858","pdf_url":"https://arxiv.org/pdf/2001.01858","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3000712052","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2001.01858","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2001.01858","doi":"https://doi.org/10.48550/arxiv.2001.01858","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2001.01858","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2001.01858","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2001.01858","pdf_url":"https://arxiv.org/pdf/2001.01858","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3000712052.pdf","grobid_xml":"https://content.openalex.org/works/W3000712052.grobid-xml"},"referenced_works_count":4,"referenced_works":["https://openalex.org/W2332506150","https://openalex.org/W2914989766","https://openalex.org/W2951947775","https://openalex.org/W2962843773"],"related_works":["https://openalex.org/W2894112326","https://openalex.org/W2997111522","https://openalex.org/W3207553910","https://openalex.org/W3207927582","https://openalex.org/W4381327335","https://openalex.org/W2900901189","https://openalex.org/W3048346032","https://openalex.org/W4285815064","https://openalex.org/W7128679117","https://openalex.org/W4399985595"],"abstract_inverted_index":{"Training":[0],"deep":[1],"learning":[2],"(DL)":[3],"models":[4],"on":[5,84],"petascale":[6],"datasets":[7],"is":[8],"essential":[9],"for":[10,33],"achieving":[11],"competitive":[12],"and":[13,23,37,55,61,80,94,100],"state-of-the-art":[14],"performance":[15,74],"in":[16],"applications":[17],"such":[18],"as":[19],"speech,":[20],"video":[21],"analytics,":[22],"object":[24],"recognition.":[25],"However,":[26],"existing":[27],"distributed":[28],"filesystems":[29],"were":[30],"not":[31],"developed":[32],"the":[34],"access":[35,66],"patterns":[36],"usability":[38],"requirements":[39],"of":[40,87],"DL":[41],"jobs.":[42],"In":[43],"this":[44],"paper,":[45],"we":[46],"describe":[47],"AIStore,":[48],"a":[49,57,85],"highly":[50],"scalable,":[51],"easy-to-deploy":[52],"storage":[53,59],"system,":[54],"WebDataset,":[56],"standards-based":[58],"format":[60],"library":[62],"that":[63],"permits":[64],"efficient":[65],"to":[67],"very":[68],"large":[69],"datasets.":[70],"We":[71],"compare":[72],"system":[73],"experimentally":[75],"using":[76],"image":[77],"classification":[78],"workloads":[79],"storing":[81],"training":[82],"data":[83],"variety":[86],"backends,":[88],"including":[89],"local":[90],"SSDs,":[91],"single-node":[92],"NFS,":[93],"two":[95],"identical":[96],"bare-metal":[97],"clusters:":[98],"HDFS":[99],"AIStore.":[101]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3000712052","counts_by_year":[{"year":2026,"cited_by_count":18},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":13},{"year":2022,"cited_by_count":10},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":1}],"updated_date":"2026-11-19T14:11:59.513116","created_date":"2020-01-23T00:00:00"}
