{"id":"https://openalex.org/W7162402361","doi":"https://doi.org/10.1145/3788853.3803085","title":"FuxiShuffle: An Adaptive and Resilient Shuffle Service for Distributed Data Processing on Alibaba Cloud","display_name":"FuxiShuffle: An Adaptive and Resilient Shuffle Service for Distributed Data Processing on Alibaba Cloud","publication_year":2026,"publication_date":"2026-05-26","ids":{"openalex":"https://openalex.org/W7162402361","doi":"https://doi.org/10.1145/3788853.3803085"},"language":null,"primary_location":{"id":"doi:10.1145/3788853.3803085","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3788853.3803085","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion of the International Conference on Management of Data","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3788853.3803085","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137077440","display_name":"Yuhao Lin","orcid":"https://orcid.org/0009-0008-7636-4376"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuhao Lin","raw_affiliation_strings":["School of Computer Science, Wuhan University, Wuhan, China and Alibaba Cloud, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0008-7636-4376","affiliations":[{"raw_affiliation_string":"School of Computer Science, Wuhan University, Wuhan, China and Alibaba Cloud, Beijing, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127277473","display_name":"Zhipeng Tang","orcid":null},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhipeng Tang","raw_affiliation_strings":["Alibaba Cloud, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0008-6957-0673","affiliations":[{"raw_affiliation_string":"Alibaba Cloud, Beijing, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058240696","display_name":"Jiayan Tong","orcid":null},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiayan Tong","raw_affiliation_strings":["Alibaba Cloud, Hangzhou, China"],"raw_orcid":"https://orcid.org/0009-0006-3236-0838","affiliations":[{"raw_affiliation_string":"Alibaba Cloud, Hangzhou, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046613099","display_name":"Junqing Xiao","orcid":null},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junqing Xiao","raw_affiliation_strings":["Alibaba Cloud, Hangzhou, China"],"raw_orcid":"https://orcid.org/0009-0006-2022-2817","affiliations":[{"raw_affiliation_string":"Alibaba Cloud, Hangzhou, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137060567","display_name":"Bin Lu","orcid":"https://orcid.org/0009-0006-6319-8597"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Lu","raw_affiliation_strings":["Alibaba Cloud, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0006-6319-8597","affiliations":[{"raw_affiliation_string":"Alibaba Cloud, Beijing, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137023304","display_name":"Yuhang Li","orcid":"https://orcid.org/0009-0002-0402-4491"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuhang Li","raw_affiliation_strings":["Alibaba Cloud, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0002-0402-4491","affiliations":[{"raw_affiliation_string":"Alibaba Cloud, Beijing, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137083915","display_name":"Chao Li","orcid":"https://orcid.org/0009-0004-4184-8806"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chao Li","raw_affiliation_strings":["Alibaba Cloud, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0004-4184-8806","affiliations":[{"raw_affiliation_string":"Alibaba Cloud, Beijing, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100363342","display_name":"Zhiguo Zhang","orcid":"https://orcid.org/0000-0001-6920-0471"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiguo Zhang","raw_affiliation_strings":["Alibaba Cloud, Hangzhou, China"],"raw_orcid":"https://orcid.org/0009-0009-5589-2601","affiliations":[{"raw_affiliation_string":"Alibaba Cloud, Hangzhou, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137069785","display_name":"Junhua Wang","orcid":"https://orcid.org/0009-0009-3260-0928"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junhua Wang","raw_affiliation_strings":["Alibaba Cloud, Hangzhou, China"],"raw_orcid":"https://orcid.org/0009-0009-3260-0928","affiliations":[{"raw_affiliation_string":"Alibaba Cloud, Hangzhou, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137047666","display_name":"Hao Luo","orcid":"https://orcid.org/0009-0001-9528-1786"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Luo","raw_affiliation_strings":["School of Computer Science, Wuhan University, Wuhan, China"],"raw_orcid":"https://orcid.org/0009-0001-9528-1786","affiliations":[{"raw_affiliation_string":"School of Computer Science, Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113409160","display_name":"James Cheng","orcid":null},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"James Cheng","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-6313-6288","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038334756","display_name":"Chuang Hu","orcid":"https://orcid.org/0000-0002-9051-3242"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chuang Hu","raw_affiliation_strings":["School of Computer Science, Wuhan University, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-9051-3242","affiliations":[{"raw_affiliation_string":"School of Computer Science, Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102918834","display_name":"Jiawei Jiang","orcid":"https://orcid.org/0000-0003-0051-0046"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiawei Jiang","raw_affiliation_strings":["School of Computer Science, Wuhan University, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0003-0051-0046","affiliations":[{"raw_affiliation_string":"School of Computer Science, Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5106729977","display_name":"Xiao Yan","orcid":null},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiao Yan","raw_affiliation_strings":["Institute for Math & AI, Wuhan, Wuhan University, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-2122-915X","affiliations":[{"raw_affiliation_string":"Institute for Math & AI, Wuhan, Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":14,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.86176058,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"386","last_page":"399"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.5759999752044678,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.5759999752044678,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.06960000097751617,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.02459999918937683,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.6111999750137329},{"id":"https://openalex.org/keywords/data-processing","display_name":"Data processing","score":0.5386000275611877},{"id":"https://openalex.org/keywords/service","display_name":"Service (business)","score":0.46230000257492065},{"id":"https://openalex.org/keywords/data-processing-system","display_name":"Data processing system","score":0.3328000009059906},{"id":"https://openalex.org/keywords/distributed-database","display_name":"Distributed database","score":0.31040000915527344},{"id":"https://openalex.org/keywords/service-provider","display_name":"Service provider","score":0.30399999022483826}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6790000200271606},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.6111999750137329},{"id":"https://openalex.org/C138827492","wikidata":"https://www.wikidata.org/wiki/Q6661985","display_name":"Data processing","level":2,"score":0.5386000275611877},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4693000018596649},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.46230000257492065},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.33970001339912415},{"id":"https://openalex.org/C112118009","wikidata":"https://www.wikidata.org/wiki/Q4925872","display_name":"Data processing system","level":2,"score":0.3328000009059906},{"id":"https://openalex.org/C70061542","wikidata":"https://www.wikidata.org/wiki/Q989016","display_name":"Distributed database","level":2,"score":0.31040000915527344},{"id":"https://openalex.org/C116537","wikidata":"https://www.wikidata.org/wiki/Q2169973","display_name":"Service provider","level":3,"score":0.30399999022483826},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2996000051498413},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.28369998931884766},{"id":"https://openalex.org/C87868495","wikidata":"https://www.wikidata.org/wiki/Q750843","display_name":"Information processing","level":2,"score":0.2741999924182892},{"id":"https://openalex.org/C557945733","wikidata":"https://www.wikidata.org/wiki/Q389772","display_name":"Data transmission","level":2,"score":0.25769999623298645}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3788853.3803085","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3788853.3803085","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion of the International Conference on Management of Data","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3788853.3803085","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3788853.3803085","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion of the International Conference on Management of Data","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W2040728701","https://openalex.org/W2049205682","https://openalex.org/W2062791769","https://openalex.org/W2067051372","https://openalex.org/W2076164405","https://openalex.org/W2118676041","https://openalex.org/W2119400430","https://openalex.org/W2173213060","https://openalex.org/W2289698129","https://openalex.org/W2322331657","https://openalex.org/W2340103561","https://openalex.org/W2414796453","https://openalex.org/W2493518685","https://openalex.org/W2550585696","https://openalex.org/W2605162773","https://openalex.org/W2735413012","https://openalex.org/W2787779818","https://openalex.org/W2798457757","https://openalex.org/W2898268271","https://openalex.org/W2929462679","https://openalex.org/W2962622685","https://openalex.org/W2963601856","https://openalex.org/W2965018236","https://openalex.org/W2970978109","https://openalex.org/W2988089807","https://openalex.org/W3028627856","https://openalex.org/W3040857534","https://openalex.org/W3048345275","https://openalex.org/W3084687783","https://openalex.org/W3086896769","https://openalex.org/W3159219445","https://openalex.org/W3197515312","https://openalex.org/W4241298546","https://openalex.org/W4285495532","https://openalex.org/W4382317604","https://openalex.org/W4392620569","https://openalex.org/W4392919237","https://openalex.org/W4398234487","https://openalex.org/W4401974846","https://openalex.org/W4411374297","https://openalex.org/W4414241821"],"related_works":[],"abstract_inverted_index":{"Shuffle":[0],"exchanges":[1],"intermediate":[2],"results":[3],"between":[4],"upstream":[5],"and":[6,13,26,61,65,72,84,103,128,160,205],"downstream":[7,126],"operators":[8],"in":[9,202],"distributed":[10],"data":[11,90,139,149],"processing":[12],"is":[14],"usually":[15],"the":[16,94,115,131],"bottleneck":[17],"due":[18],"to":[19,34,57,108,177],"factors":[20],"such":[21],"as":[22,87],"small":[23],"random":[24],"I/Os":[25],"network":[27],"contention.":[28],"Several":[29],"systems":[30],"have":[31],"been":[32],"designed":[33],"improve":[35],"shuffle":[36,91,116,138],"efficiency,":[37],"but":[38,189],"from":[39],"our":[40,198],"experiences":[41],"of":[42,98],"running":[43],"ultra-large":[44,95],"clusters":[45],"at":[46],"Alibaba":[47],"Cloud":[48],"MaxCompute,":[49,99],"we":[50,82],"observe":[51],"that":[52,166,197],"they":[53],"can":[54],"not":[55,168,183],"adapt":[56],"highly":[58],"dynamic":[59],"workloads":[60],"cluster":[62],"resource":[63,192],"conditions,":[64],"their":[66,80],"fault":[67],"tolerance":[68],"mechanisms":[69],"are":[70,76,200],"passive":[71],"inefficient":[73],"when":[74],"failures":[75],"inevitable.":[77],"To":[78,141],"tackle":[79],"limitations,":[81],"design":[83],"implement":[85],"FuxiShuffle":[86,112,146,180],"a":[88],"general":[89],"service":[92],"for":[93,125,136],"production":[96],"environment":[97],"featuring":[100],"good":[101,110],"adaptability":[102,204],"efficient":[104],"failure":[105,143],"resilience.":[106,206],"Specifically,":[107],"achieve":[109],"adaptability,":[111],"dynamically":[113],"selects":[114],"mode":[117],"based":[118],"on":[119],"runtime":[120],"information,":[121],"conducts":[122],"progress-aware":[123],"scheduling":[124],"workers,":[127],"automatically":[129],"determines":[130],"most":[132],"suitable":[133],"backup":[134],"strategy":[135],"each":[137],"chunk.":[140],"make":[142],"resilience":[144],"efficient,":[145],"actively":[147],"ensures":[148],"availability":[150],"with":[151,156],"multi-replica":[152],"failover,":[153],"prevents":[154],"overflow":[155],"careful":[157],"memory":[158],"management,":[159],"employs":[161],"an":[162],"incremental":[163],"recovery":[164],"mechanism":[165],"does":[167],"lose":[169],"computation":[170],"progress.":[171],"Our":[172],"experiments":[173,195],"show":[174],"that,":[175],"compared":[176],"baseline":[178],"systems,":[179],"significantly":[181],"reduces":[182],"only":[184],"end-to-end":[185],"job":[186],"completion":[187],"time":[188],"also":[190],"aggregate":[191],"consumption.":[193],"Micro":[194],"suggest":[196],"designs":[199],"effective":[201],"improving":[203]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-05-27T00:00:00"}