{"id":"https://openalex.org/W4414241786","doi":"https://doi.org/10.14778/3750601.3750620","title":"Magnus: A Holistic Approach to Data Management for Large-Scale Machine Learning Workloads","display_name":"Magnus: A Holistic Approach to Data Management for Large-Scale Machine Learning Workloads","publication_year":2025,"publication_date":"2025-08-01","ids":{"openalex":"https://openalex.org/W4414241786","doi":"https://doi.org/10.14778/3750601.3750620"},"language":"en","primary_location":{"id":"doi:10.14778/3750601.3750620","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3750601.3750620","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5112705359","display_name":"Jun Hyuk Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jun Song","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110081731","display_name":"Jau M. Ding","orcid":null},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingyi Ding","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119634054","display_name":"Irshad Kandy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Irshad Kandy","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109500455","display_name":"Yanghao Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yanghao Lin","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhongjia Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhongjia Wei","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087107561","display_name":"Zilong Zhou","orcid":"https://orcid.org/0009-0002-1655-5089"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zilong Zhou","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061729313","display_name":"Zhiwei Peng","orcid":"https://orcid.org/0000-0003-1720-0749"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhiwei Peng","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020311129","display_name":"Shan Jixi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jixi Shan","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035681554","display_name":"Hongyue Mao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hongyue Mao","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035676193","display_name":"Xiuqi Huang","orcid":"https://orcid.org/0009-0004-4443-1555"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiuqi Huang","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xun Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xun Song","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100420548","display_name":"Cheng Chen","orcid":"https://orcid.org/0000-0002-4203-2145"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng Chen","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033930067","display_name":"Yanjia Li","orcid":"https://orcid.org/0009-0003-9498-1216"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yanjia Li","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003867687","display_name":"Tianhao Yang","orcid":"https://orcid.org/0000-0001-9820-5087"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tianhao Yang","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074215832","display_name":"Jia Wei","orcid":"https://orcid.org/0000-0001-5564-2172"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei Jia","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101856003","display_name":"Xiaohong Dong","orcid":"https://orcid.org/0000-0002-7047-4864"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiaohong Dong","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044025660","display_name":"Lei Kang","orcid":"https://orcid.org/0000-0002-1962-3916"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kang Lei","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048460408","display_name":"Rui Shi","orcid":"https://orcid.org/0009-0001-6471-6759"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rui Shi","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112302242","display_name":"Pengwei Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pengwei Zhao","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100690338","display_name":"Wei Chen","orcid":"https://orcid.org/0000-0001-5429-2844"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Chen","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.8166,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.91897914,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"18","issue":"12","first_page":"4964","last_page":"4977"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9929999709129333,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9929999709129333,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9912999868392944,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.8364999890327454},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7010999917984009},{"id":"https://openalex.org/keywords/data-management","display_name":"Data management","score":0.5914999842643738},{"id":"https://openalex.org/keywords/support-vector-machine","display_name":"Support vector machine","score":0.48019999265670776},{"id":"https://openalex.org/keywords/data-modeling","display_name":"Data modeling","score":0.42969998717308044},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.364300012588501},{"id":"https://openalex.org/keywords/online-machine-learning","display_name":"Online machine learning","score":0.3513999879360199},{"id":"https://openalex.org/keywords/data-integration","display_name":"Data integration","score":0.3375999927520752},{"id":"https://openalex.org/keywords/metadata-management","display_name":"Metadata management","score":0.33340001106262207}],"concepts":[{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.8364999890327454},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7907000184059143},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7010999917984009},{"id":"https://openalex.org/C1668388","wikidata":"https://www.wikidata.org/wiki/Q1149776","display_name":"Data management","level":2,"score":0.5914999842643738},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5142999887466431},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.48019999265670776},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.42969998717308044},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41290000081062317},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.36570000648498535},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.364300012588501},{"id":"https://openalex.org/C115903097","wikidata":"https://www.wikidata.org/wiki/Q7094097","display_name":"Online machine learning","level":3,"score":0.3513999879360199},{"id":"https://openalex.org/C72634772","wikidata":"https://www.wikidata.org/wiki/Q386824","display_name":"Data integration","level":2,"score":0.3375999927520752},{"id":"https://openalex.org/C2779489174","wikidata":"https://www.wikidata.org/wiki/Q6822246","display_name":"Metadata management","level":3,"score":0.33340001106262207},{"id":"https://openalex.org/C2777851325","wikidata":"https://www.wikidata.org/wiki/Q7094102","display_name":"Online model","level":2,"score":0.33250001072883606},{"id":"https://openalex.org/C888380","wikidata":"https://www.wikidata.org/wiki/Q2427787","display_name":"Persistent data structure","level":2,"score":0.32409998774528503},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3181000053882599},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3158999979496002},{"id":"https://openalex.org/C110326360","wikidata":"https://www.wikidata.org/wiki/Q17149476","display_name":"Metadata modeling","level":4,"score":0.30390000343322754},{"id":"https://openalex.org/C100463513","wikidata":"https://www.wikidata.org/wiki/Q5227322","display_name":"Data model (GIS)","level":2,"score":0.28929999470710754},{"id":"https://openalex.org/C13926793","wikidata":"https://www.wikidata.org/wiki/Q3507155","display_name":"Group method of data handling","level":2,"score":0.2833000123500824},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.2786000072956085},{"id":"https://openalex.org/C138827492","wikidata":"https://www.wikidata.org/wiki/Q6661985","display_name":"Data processing","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C2779965156","wikidata":"https://www.wikidata.org/wiki/Q5227350","display_name":"Data sharing","level":3,"score":0.2718999981880188},{"id":"https://openalex.org/C137314826","wikidata":"https://www.wikidata.org/wiki/Q2330408","display_name":"Data mapping","level":2,"score":0.2705000042915344},{"id":"https://openalex.org/C3102742","wikidata":"https://www.wikidata.org/wiki/Q6031177","display_name":"Information repository","level":3,"score":0.2644999921321869},{"id":"https://openalex.org/C138958017","wikidata":"https://www.wikidata.org/wiki/Q190087","display_name":"Data type","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C551230270","wikidata":"https://www.wikidata.org/wiki/Q4368942","display_name":"Data retrieval","level":2,"score":0.2578999996185303},{"id":"https://openalex.org/C2780616401","wikidata":"https://www.wikidata.org/wiki/Q1133673","display_name":"Cornerstone","level":2,"score":0.2567000091075897},{"id":"https://openalex.org/C206175624","wikidata":"https://www.wikidata.org/wiki/Q595731","display_name":"Branching (polymer chemistry)","level":2,"score":0.25589999556541443},{"id":"https://openalex.org/C2778484313","wikidata":"https://www.wikidata.org/wiki/Q1172540","display_name":"Data stream","level":2,"score":0.2524999976158142},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2524000108242035},{"id":"https://openalex.org/C61871575","wikidata":"https://www.wikidata.org/wiki/Q384093","display_name":"Master data","level":2,"score":0.2522999942302704},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2506999969482422}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.14778/3750601.3750620","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3750601.3750620","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W3085477028","https://openalex.org/W3085940077","https://openalex.org/W3198412807","https://openalex.org/W4281657584","https://openalex.org/W4282585030","https://openalex.org/W4296415418","https://openalex.org/W4367032190","https://openalex.org/W4384652670","https://openalex.org/W4385565675","https://openalex.org/W4386768921","https://openalex.org/W4398234406","https://openalex.org/W4398234408","https://openalex.org/W4400909549","https://openalex.org/W4403747745","https://openalex.org/W4404181225","https://openalex.org/W4404181247","https://openalex.org/W4413360297"],"related_works":[],"abstract_inverted_index":{"Machine":[0],"learning":[1],"(ML)":[2],"has":[3,144],"become":[4],"a":[5,65],"cornerstone":[6],"of":[7],"key":[8],"applications":[9],"at":[10,147],"ByteDance.":[11],"As":[12],"model":[13],"complexity":[14],"and":[15,37,53,87,94,107,110,124,129,155],"data":[16,19,67,99,157],"volumes":[17],"surge,":[18],"management":[20,68],"for":[21,83,92,127,149,159],"large-scale":[22,160],"ML":[23,56,141,161],"workloads":[24],"faces":[25],"substantial":[26],"challenges,":[27,61],"particularly":[28],"with":[29,55,104],"recent":[30],"advances":[31],"in":[32,46,139],"large":[33,38,84],"recommendation":[34],"models":[35,40],"(LRMs)":[36],"multimodal":[39,88],"(LMMs).":[41],"Traditional":[42],"approaches":[43],"exhibit":[44],"limitations":[45],"storage":[47,80],"efficiency,":[48],"metadata":[49,102],"scalability,":[50],"update":[51],"mechanisms,":[52],"integration":[54],"frameworks.":[57],"To":[58],"address":[59],"these":[60],"we":[62],"propose":[63],"Magnus,":[64],"holistic":[66],"system":[69],"built":[70],"upon":[71],"Apache":[72],"Iceberg.":[73],"Magnus":[74,120,143],"integrates":[75],"innovative":[76],"optimizations":[77],"across":[78],"resource-efficient":[79],"formats":[81],"optimized":[82],"wide":[85],"tables":[86],"data,":[89],"built-in":[90],"support":[91,123],"vector":[93],"inverted":[95],"indexes":[96],"to":[97],"accelerate":[98],"retrieval,":[100],"scalable":[101],"planning":[103],"Git-like":[105],"branching":[106],"tagging":[108],"capabilities,":[109],"high-performance":[111],"update/upsert":[112],"based":[113],"on":[114],"lightweight":[115],"merge-on-read":[116],"(MOR)":[117],"strategies.":[118],"Additionally,":[119],"provides":[121],"native":[122],"specialized":[125],"enhancement":[126],"LRM":[128],"LMM":[130],"training":[131],"workloads.":[132,162],"Experimental":[133],"results":[134],"demonstrate":[135],"significant":[136],"performance":[137],"gains":[138],"real-world":[140],"scenarios.":[142],"been":[145],"deployed":[146],"ByteDance":[148],"over":[150],"five":[151],"years,":[152],"enabling":[153],"robust":[154],"efficient":[156],"infrastructure":[158]},"counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}