{"id":"https://openalex.org/W7166734877","doi":"https://doi.org/10.48550/arxiv.2606.29975","title":"Atompack: A Storage and Distribution Layer for Read-Heavy Atomistic ML Training Datasets","display_name":"Atompack: A Storage and Distribution Layer for Read-Heavy Atomistic ML Training Datasets","publication_year":2026,"publication_date":"2026-06-29","ids":{"openalex":"https://openalex.org/W7166734877","doi":"https://doi.org/10.48550/arxiv.2606.29975"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.29975","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29975","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.29975","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104435508","display_name":"Ali Ramlaoui","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ramlaoui, Ali","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139640978","display_name":"Daniel T. Speckhard","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Speckhard, Daniel T.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018013133","display_name":"Sagar Pal","orcid":"https://orcid.org/0000-0002-3425-7010"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pal, Sagar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139669079","display_name":"Fragkiskos D. Malliaros","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Malliaros, Fragkiskos D.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083981921","display_name":"Alexandre Duval","orcid":"https://orcid.org/0000-0001-9416-3270"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Duval, Alexandre","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136722708","display_name":"Victor Schmidt","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schmidt, Victor","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9223999977111816,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9223999977111816,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.03319999948143959,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10857","display_name":"Advanced Electron Microscopy Techniques and Applications","score":0.002300000051036477,"subfield":{"id":"https://openalex.org/subfields/1315","display_name":"Structural Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5720999836921692},{"id":"https://openalex.org/keywords/artifact","display_name":"Artifact (error)","score":0.5716999769210815},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.5112000107765198},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.4968000054359436},{"id":"https://openalex.org/keywords/path","display_name":"Path (computing)","score":0.49559998512268066},{"id":"https://openalex.org/keywords/pipeline-transport","display_name":"Pipeline transport","score":0.47110000252723694},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.46219998598098755},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.4593000113964081},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.41830000281333923}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7175999879837036},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5720999836921692},{"id":"https://openalex.org/C2779010991","wikidata":"https://www.wikidata.org/wiki/Q2720909","display_name":"Artifact (error)","level":2,"score":0.5716999769210815},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.5112000107765198},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.4968000054359436},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.49559998512268066},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4794999957084656},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.47110000252723694},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.46219998598098755},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4593000113964081},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.41830000281333923},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4113999903202057},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3984000086784363},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.3944000005722046},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.38339999318122864},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.375},{"id":"https://openalex.org/C2778751112","wikidata":"https://www.wikidata.org/wiki/Q835016","display_name":"Window (computing)","level":2,"score":0.36079999804496765},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.31459999084472656},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3005000054836273},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.29330000281333923},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.28760001063346863},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.28760001063346863},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.2856999933719635},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.2732999920845032},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.26010000705718994},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.2597000002861023},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2578999996185303},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.2554999887943268}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.29975","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29975","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.29975","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29975","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Atomistic":[0],"machine":[1],"learning":[2,82],"datasets":[3],"are":[4,12,43],"increasingly":[5],"used":[6],"for":[7,105,186],"training:":[8],"large":[9],"immutable":[10,94],"snapshots":[11],"read":[13,102],"repeatedly,":[14],"shuffled":[15,131,153,178],"across":[16,19],"epochs,":[17],"staged":[18],"clusters'":[20],"storage":[21,56],"systems,":[22],"and":[23,39,58,96,113,123,137],"republished":[24],"as":[25],"reusable":[26],"scientific":[27,34],"artifacts.":[28],"This":[29],"workload":[30],"differs":[31],"from":[32],"interactive":[33],"curation,":[35],"where":[36],"mutable":[37],"records":[38,77,86,98],"ad":[40],"hoc":[41],"inspection":[42],"often":[44],"more":[45],"important":[46],"than":[47,149,171],"random":[48],"indexed":[49],"throughput.":[50],"We":[51,107],"present":[52],"Atompack,":[53],"an":[54,93],"append-oriented":[55],"format":[57],"distribution":[59],"layer":[60],"designed":[61],"around":[62],"a":[63,100,141],"simple":[64],"workload:":[65],"training":[66,179],"pipelines":[67],"usually":[68],"consume":[69],"complete":[70,167],"molecular":[71],"records,":[72,120,122,169],"while":[73,156,181],"the":[74,81],"order":[75],"of":[76],"is":[78,146],"randomized":[79],"by":[80],"algorithm.":[83],"Atompack":[84,109,145],"appends":[85],"efficiently":[87],"during":[88],"dataset":[89],"construction,":[90],"then":[91],"commits":[92],"index":[95],"serves":[97],"through":[99],"memory-mapped":[101],"path":[103],"optimized":[104],"training.":[106],"compare":[108],"with":[110],"HDF5,":[111],"LMDB,":[112],"ASE":[114,150],"baselines":[115],"representing":[116],"array":[117],"stores,":[118],"key-value":[119],"serialized":[121],"object-oriented":[124],"databases.":[125],"The":[126,162],"benchmarks":[127],"measure":[128],"sequential":[129],"reads,":[130,132],"shared-filesystem":[133],"behavior,":[134],"write":[135],"throughput,":[136],"artifact":[138],"size.":[139],"On":[140],"representative":[142],"64-atom":[143],"workload,":[144],"96x":[147],"faster":[148],"LMDB":[151],"on":[152],"training-style":[154],"reads":[155],"producing":[157],"artifacts":[158,183],"about":[159],"79\\%":[160],"smaller.":[161],"results":[163],"indicate":[164],"that":[165],"serving":[166],"molecule":[168],"rather":[170],"field":[172],"chunks":[173],"or":[174],"reconstructed":[175],"objects,":[176],"improves":[177],"throughput":[180],"keeping":[182],"compact":[184],"enough":[185],"public":[187],"distribution.":[188]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-01T00:00:00"}