{"id":"https://openalex.org/W4412623564","doi":"https://doi.org/10.1109/tsc.2025.3592427","title":"MPDA: A Massively Parallel Learning and Dependency-Aware Scheduling Algorithm for Data Processing Clusters","display_name":"MPDA: A Massively Parallel Learning and Dependency-Aware Scheduling Algorithm for Data Processing Clusters","publication_year":2025,"publication_date":"2025-07-24","ids":{"openalex":"https://openalex.org/W4412623564","doi":"https://doi.org/10.1109/tsc.2025.3592427"},"language":"en","primary_location":{"id":"doi:10.1109/tsc.2025.3592427","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tsc.2025.3592427","pdf_url":null,"source":{"id":"https://openalex.org/S204223317","display_name":"IEEE Transactions on Services Computing","issn_l":"1939-1374","issn":["1939-1374","2372-0204"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Services Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100734609","display_name":"Qing Li","orcid":"https://orcid.org/0000-0002-6071-473X"},"institutions":[{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Qing Li","raw_affiliation_strings":["Pengcheng Laboratory, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-6071-473X","affiliations":[{"raw_affiliation_string":"Pengcheng Laboratory, Shenzhen, China","institution_ids":["https://openalex.org/I4210136793"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046610531","display_name":"Xingchi Chen","orcid":"https://orcid.org/0009-0007-6296-5122"},"institutions":[{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xingchi Chen","raw_affiliation_strings":["Pengcheng Laboratory, Shenzhen, China"],"raw_orcid":"https://orcid.org/0009-0007-6296-5122","affiliations":[{"raw_affiliation_string":"Pengcheng Laboratory, Shenzhen, China","institution_ids":["https://openalex.org/I4210136793"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107897271","display_name":"J. Guo","orcid":"https://orcid.org/0000-0001-8125-9433"},"institutions":[{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiawei Guo","raw_affiliation_strings":["Tsinghua Shenzhen International Graduate School, Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, Shenzhen, China","institution_ids":["https://openalex.org/I4210114105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085581846","display_name":"Fa Zhu","orcid":"https://orcid.org/0000-0002-7089-5386"},"institutions":[{"id":"https://openalex.org/I167027274","display_name":"Nanjing Forestry University","ror":"https://ror.org/03m96p165","country_code":"CN","type":"education","lineage":["https://openalex.org/I167027274"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fa Zhu","raw_affiliation_strings":["College of Information Science and Technology, Nanjing Forestry University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-7089-5386","affiliations":[{"raw_affiliation_string":"College of Information Science and Technology, Nanjing Forestry University, Nanjing, China","institution_ids":["https://openalex.org/I167027274"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033912893","display_name":"Achyut Shankar","orcid":"https://orcid.org/0000-0003-3165-3293"},"institutions":[{"id":"https://openalex.org/I3129773123","display_name":"Bennett University","ror":"https://ror.org/00an5hx75","country_code":"IN","type":"education","lineage":["https://openalex.org/I3129773123"]},{"id":"https://openalex.org/I39555362","display_name":"University of Warwick","ror":"https://ror.org/01a77tt86","country_code":"GB","type":"education","lineage":["https://openalex.org/I39555362"]}],"countries":["GB","IN"],"is_corresponding":false,"raw_author_name":"Achyut Shankar","raw_affiliation_strings":["School of Computer Science Engineering and Technology, Bennett University, Greater Noida, Uttar Pradesh, India","Department of Cyber Systems Engineering, University of Warwick, Coventry, U.K"],"raw_orcid":"https://orcid.org/0000-0003-3165-3293","affiliations":[{"raw_affiliation_string":"School of Computer Science Engineering and Technology, Bennett University, Greater Noida, Uttar Pradesh, India","institution_ids":["https://openalex.org/I3129773123"]},{"raw_affiliation_string":"Department of Cyber Systems Engineering, University of Warwick, Coventry, U.K","institution_ids":["https://openalex.org/I39555362"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075393874","display_name":"Fayez Alqahtani","orcid":"https://orcid.org/0000-0001-8972-5953"},"institutions":[{"id":"https://openalex.org/I28022161","display_name":"King Saud University","ror":"https://ror.org/02f81g417","country_code":"SA","type":"education","lineage":["https://openalex.org/I28022161"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Fayez Alqahtani","raw_affiliation_strings":["Software Engineering Department, College of Computer and Information Sciences, King Saud University, Riyadh, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0001-8972-5953","affiliations":[{"raw_affiliation_string":"Software Engineering Department, College of Computer and Information Sciences, King Saud University, Riyadh, Saudi Arabia","institution_ids":["https://openalex.org/I28022161"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071657438","display_name":"Kamalakanta Muduli","orcid":"https://orcid.org/0000-0002-4245-9149"},"institutions":[{"id":"https://openalex.org/I119444370","display_name":"Papua New Guinea University of Technology","ror":"https://ror.org/012g45386","country_code":"PG","type":"education","lineage":["https://openalex.org/I119444370"]}],"countries":["PG"],"is_corresponding":false,"raw_author_name":"Kamalakanta Muduli","raw_affiliation_strings":["Papua New Guinea University of Technology, Lae, Morobe Province, Papua New Guinea","Papua New Guinea University of Technology Lae, Morobe, Papua New Guinea"],"raw_orcid":"https://orcid.org/0000-0002-4245-9149","affiliations":[{"raw_affiliation_string":"Papua New Guinea University of Technology, Lae, Morobe Province, Papua New Guinea","institution_ids":["https://openalex.org/I119444370"]},{"raw_affiliation_string":"Papua New Guinea University of Technology Lae, Morobe, Papua New Guinea","institution_ids":["https://openalex.org/I119444370"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087021865","display_name":"Bo Yi","orcid":"https://orcid.org/0000-0002-8073-8202"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bo Yi","raw_affiliation_strings":["College of Computer Science and Engineering, Northeastern University, Shenyang, China"],"raw_orcid":"https://orcid.org/0000-0002-8073-8202","affiliations":[{"raw_affiliation_string":"College of Computer Science and Engineering, Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101626204","display_name":"Yong Jiang","orcid":"https://orcid.org/0000-0002-4260-1395"},"institutions":[{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yong Jiang","raw_affiliation_strings":["Tsinghua Shenzhen International Graduate School, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-4260-1395","affiliations":[{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, Shenzhen, China","institution_ids":["https://openalex.org/I4210114105"]}]}],"institutions":[],"countries_distinct_count":5,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5100734609"],"corresponding_institution_ids":["https://openalex.org/I4210136793"],"apc_list":null,"apc_paid":null,"fwci":1.3335,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.83745715,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"18","issue":"5","first_page":"3038","last_page":"3051"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.953499972820282,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.953499972820282,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9422000050544739,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8788858652114868},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.6294696927070618},{"id":"https://openalex.org/keywords/massively-parallel","display_name":"Massively parallel","score":0.6073102355003357},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.5273923277854919},{"id":"https://openalex.org/keywords/dependency","display_name":"Dependency (UML)","score":0.46499624848365784},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3834070563316345},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.369751900434494},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.21568968892097473}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8788858652114868},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6294696927070618},{"id":"https://openalex.org/C190475519","wikidata":"https://www.wikidata.org/wiki/Q544384","display_name":"Massively parallel","level":2,"score":0.6073102355003357},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5273923277854919},{"id":"https://openalex.org/C19768560","wikidata":"https://www.wikidata.org/wiki/Q320727","display_name":"Dependency (UML)","level":2,"score":0.46499624848365784},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3834070563316345},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.369751900434494},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21568968892097473},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tsc.2025.3592427","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tsc.2025.3592427","pdf_url":null,"source":{"id":"https://openalex.org/S204223317","display_name":"IEEE Transactions on Services Computing","issn_l":"1939-1374","issn":["1939-1374","2372-0204"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Services Computing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6027343913","display_name":null,"funder_award_id":"61972189","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":40,"referenced_works":["https://openalex.org/W626266683","https://openalex.org/W2022678927","https://openalex.org/W2025549137","https://openalex.org/W2038412523","https://openalex.org/W2055748525","https://openalex.org/W2096125134","https://openalex.org/W2100890245","https://openalex.org/W2102849319","https://openalex.org/W2110856814","https://openalex.org/W2116341502","https://openalex.org/W2145339207","https://openalex.org/W2148204686","https://openalex.org/W2149931128","https://openalex.org/W2169209873","https://openalex.org/W2335814492","https://openalex.org/W2546571074","https://openalex.org/W2548753368","https://openalex.org/W2766447205","https://openalex.org/W2774053434","https://openalex.org/W2793081219","https://openalex.org/W2807796354","https://openalex.org/W2907492528","https://openalex.org/W2919897868","https://openalex.org/W2968986602","https://openalex.org/W2982346115","https://openalex.org/W2984408162","https://openalex.org/W3010499512","https://openalex.org/W3034384938","https://openalex.org/W3035617116","https://openalex.org/W3162118826","https://openalex.org/W3181204626","https://openalex.org/W3207532613","https://openalex.org/W4319778953","https://openalex.org/W4320005491","https://openalex.org/W4320018413","https://openalex.org/W4321487996","https://openalex.org/W4378696997","https://openalex.org/W4403295007","https://openalex.org/W4403446944","https://openalex.org/W6922480057"],"related_works":["https://openalex.org/W2023839151","https://openalex.org/W1774183074","https://openalex.org/W2057488824","https://openalex.org/W2334687145","https://openalex.org/W2178011914","https://openalex.org/W4235962491","https://openalex.org/W2061778832","https://openalex.org/W1513001507","https://openalex.org/W2257153718","https://openalex.org/W3134702077"],"abstract_inverted_index":{"In":[0],"the":[1,17,86,92,95,111,120,137,140,149],"era":[2],"of":[3,39,55,94,115,139,157],"large-scale":[4],"machine":[5],"learning,":[6],"largescale":[7],"clusters":[8],"are":[9,29],"extensively":[10],"used":[11],"for":[12,107],"data":[13],"processing":[14],"jobs.":[15],"However,":[16],"state-of-the-art":[18],"heuristic-based":[19],"and":[20,37,50,59,89,101,113,129,159],"Deep":[21],"Rein-forcement":[22],"Learning":[23],"(DRL)":[24],"based":[25],"job":[26,167],"scheduling":[27,52,141,162,173],"mechanisms":[28],"facing":[30],"challenges":[31,70],"such":[32],"as":[33],"slow":[34],"training":[35,80,87,106,150],"speed":[36,151],"underexploitation":[38],"jobs'":[40],"complex":[41],"dependencies.":[42],"We":[43],"propose":[44],"MPDA,":[45,108],"a":[46,56,60,76],"Massively":[47],"Parallel":[48],"learning":[49,100],"Dependency-Aware":[51],"algorithm,":[53],"consisting":[54],"fast-training":[57,73],"mechanism":[58,74],"novel":[61],"dependency-aware":[62],"policy":[63],"network,":[64],"GATNetwork,":[65],"to":[66,135,154],"address":[67],"these":[68],"two":[69,155],"respectively.":[71],"The":[72,117,143],"is":[75],"two-level":[77],"massively":[78],"parallel":[79],"method":[81],"that":[82,146],"can":[83],"significantly":[84],"accelerate":[85],"process":[88],"maximally":[90],"utilize":[91],"resources":[93],"cluster.":[96],"Additionally,":[97],"its":[98],"decoupled":[99],"interacting":[102],"design":[103],"enables":[104],"hybrid-workload":[105],"which":[109],"guarantees":[110],"generalization":[112],"robustness":[114],"MPDA.":[116],"GATNetwork":[118],"exploits":[119],"dependencies":[121],"among":[122],"stages/jobs":[123],"using":[124],"Graph":[125],"Attention":[126],"Network":[127],"(GAT)":[128],"Long":[130],"Short-Term":[131],"Memory":[132],"(LSTM)":[133],"networks":[134],"improve":[136],"performance":[138],"policy.":[142],"experiments":[144],"show":[145],"MPDA":[147],"accelerates":[148],"by":[152],"one":[153],"orders":[156],"magnitude":[158],"achieves":[160],"better":[161],"performance,":[163],"i.e.,":[164],"lower":[165],"average":[166],"completion":[168],"time,":[169],"compared":[170],"with":[171],"existing":[172],"algorithms.":[174]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-04-29T09:16:38.111599","created_date":"2025-10-10T00:00:00"}