{"id":"https://openalex.org/W4412610356","doi":"https://doi.org/10.1109/hpcc64274.2024.00073","title":"Optimizing inter-stage communication in Spark via time-varying network bandwidth utilization","display_name":"Optimizing inter-stage communication in Spark via time-varying network bandwidth utilization","publication_year":2024,"publication_date":"2024-12-13","ids":{"openalex":"https://openalex.org/W4412610356","doi":"https://doi.org/10.1109/hpcc64274.2024.00073"},"language":"en","primary_location":{"id":"doi:10.1109/hpcc64274.2024.00073","is_oa":false,"landing_page_url":"https://doi.org/10.1109/hpcc64274.2024.00073","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on High Performance Computing and Communications (HPCC)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5035071151","display_name":"Changpeng Zhu","orcid":"https://orcid.org/0000-0002-7272-7036"},"institutions":[{"id":"https://openalex.org/I145581781","display_name":"Chongqing Technology and Business University","ror":"https://ror.org/05hqf1284","country_code":"CN","type":"education","lineage":["https://openalex.org/I145581781"]},{"id":"https://openalex.org/I168337820","display_name":"Chongqing University of Science and Technology","ror":"https://ror.org/03n3v6d52","country_code":"CN","type":"education","lineage":["https://openalex.org/I168337820"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Changpeng Zhu","raw_affiliation_strings":["Chongqing University of Technology,Department of Data Science and Big Data,Chongqing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chongqing University of Technology,Department of Data Science and Big Data,Chongqing,China","institution_ids":["https://openalex.org/I145581781","https://openalex.org/I168337820"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100442271","display_name":"Xi Wang","orcid":"https://orcid.org/0000-0002-7572-6354"},"institutions":[{"id":"https://openalex.org/I145581781","display_name":"Chongqing Technology and Business University","ror":"https://ror.org/05hqf1284","country_code":"CN","type":"education","lineage":["https://openalex.org/I145581781"]},{"id":"https://openalex.org/I168337820","display_name":"Chongqing University of Science and Technology","ror":"https://ror.org/03n3v6d52","country_code":"CN","type":"education","lineage":["https://openalex.org/I168337820"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xi Wang","raw_affiliation_strings":["Chongqing University of Technology,Department of Data Science and Big Data,Chongqing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chongqing University of Technology,Department of Data Science and Big Data,Chongqing,China","institution_ids":["https://openalex.org/I145581781","https://openalex.org/I168337820"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101936636","display_name":"Bo Han","orcid":"https://orcid.org/0000-0003-2591-5859"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bo Han","raw_affiliation_strings":["Xi’an Jiaotong University,Department of Cyber Science and Engineering,Xi’an,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi’an Jiaotong University,Department of Cyber Science and Engineering,Xi’an,China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023527761","display_name":"Tian Zhou","orcid":"https://orcid.org/0000-0001-8228-6855"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tian Zhou","raw_affiliation_strings":["Xi’an Jiaotong University,Department of Cyber Science and Engineering,Xi’an,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi’an Jiaotong University,Department of Cyber Science and Engineering,Xi’an,China","institution_ids":["https://openalex.org/I87445476"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.43708662,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"498","last_page":"507"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10829","display_name":"Interconnection Networks and Systems","score":0.9860000014305115,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10829","display_name":"Interconnection Networks and Systems","score":0.9860000014305115,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10904","display_name":"Embedded Systems Design Techniques","score":0.9484999775886536,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12216","display_name":"Network Time Synchronization Technologies","score":0.9312000274658203,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bandwidth","display_name":"Bandwidth (computing)","score":0.7070826292037964},{"id":"https://openalex.org/keywords/stage","display_name":"Stage (stratigraphy)","score":0.6668409109115601},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6242156624794006},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.5666776299476624},{"id":"https://openalex.org/keywords/dynamic-bandwidth-allocation","display_name":"Dynamic bandwidth allocation","score":0.444538414478302},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.32479941844940186}],"concepts":[{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.7070826292037964},{"id":"https://openalex.org/C146357865","wikidata":"https://www.wikidata.org/wiki/Q1123245","display_name":"Stage (stratigraphy)","level":2,"score":0.6668409109115601},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6242156624794006},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.5666776299476624},{"id":"https://openalex.org/C145062175","wikidata":"https://www.wikidata.org/wiki/Q5318947","display_name":"Dynamic bandwidth allocation","level":3,"score":0.444538414478302},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.32479941844940186},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/hpcc64274.2024.00073","is_oa":false,"landing_page_url":"https://doi.org/10.1109/hpcc64274.2024.00073","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on High Performance Computing and Communications (HPCC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/7","display_name":"Affordable and clean energy","score":0.4099999964237213}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W2105947650","https://openalex.org/W2129542763","https://openalex.org/W2155072926","https://openalex.org/W2997118694","https://openalex.org/W3015618226","https://openalex.org/W3021375466","https://openalex.org/W3109595290","https://openalex.org/W3124926625","https://openalex.org/W3126168625","https://openalex.org/W3129341457","https://openalex.org/W3173985525","https://openalex.org/W3189702920","https://openalex.org/W3198116371","https://openalex.org/W3198834566","https://openalex.org/W4251459357","https://openalex.org/W4399110491","https://openalex.org/W6687322159","https://openalex.org/W6756010882","https://openalex.org/W6773076937","https://openalex.org/W6861781400"],"related_works":["https://openalex.org/W2141211540","https://openalex.org/W2029299146","https://openalex.org/W1999486158","https://openalex.org/W2389261828","https://openalex.org/W3176276356","https://openalex.org/W2073876121","https://openalex.org/W2128382336","https://openalex.org/W2350829955","https://openalex.org/W2120509413","https://openalex.org/W2061713288"],"abstract_inverted_index":{"Spark":[0,69,190,210,227],"has":[1],"garnered":[2],"significant":[3,195],"attention":[4],"owing":[5],"to":[6,25,56,64,108,129,176,182,197,202,232,236],"its":[7,136],"superior":[8],"capability":[9],"in":[10,43,112,138,172,186,234],"processing":[11],"vast":[12],"amounts":[13],"of":[14,31,68,82,125,133,135,148,151,165,189,208],"data.":[15],"Its":[16],"executor-based":[17],"model":[18],"necessitates":[19],"that":[20,222],"sufficient":[21],"resources":[22],"are":[23,160],"allocated":[24],"executors,":[26],"thus":[27],"facilitating":[28],"the":[29,66,80,123,131,144,149,177,187,204,209,218,237],"executions":[30],"diverse":[32],"tasks.":[33],"These":[34],"tasks":[35,137,150],"often":[36,54,88],"exhibit":[37],"different":[38],"resource":[39,45,57],"consumption":[40,46],"patterns,":[41],"resulting":[42],"time-varying":[44,72,92],"concerning":[47],"CPU,":[48],"network":[49,95,120],"resources,":[50],"etc.":[51],"Consequently,":[52,155],"this":[53,99,101,116],"leads":[55,181],"wastage.":[58],"Some":[59],"research":[60],"endeavors":[61],"have":[62,169],"aimed":[63],"optimize":[65,109],"performance":[67,188,229],"applications":[70],"with":[71],"CPU":[73],"consumption.":[74],"Their":[75],"strategies":[76],"primarily":[77],"involve":[78],"adjusting":[79],"number":[81],"executors":[83],"dynamically.":[84],"However,":[85],"these":[86,156],"efforts":[87],"overlook":[89],"another":[90],"crucial":[91],"resource,":[93],"namely,":[94],"bandwidth":[96,121],"usage.To":[97],"address":[98],"issue,":[100],"paper":[102],"proposes":[103],"a":[104],"novel":[105],"pre-transmission":[106,178],"approach":[107,117,193,224],"inter-stage":[110],"communication":[111],"Spark.":[113,239],"In":[114],"essence,":[115],"leverages":[118],"idle":[119],"during":[122],"runtime":[124],"an":[126,183],"upstream":[127],"stage":[128],"initiate":[130],"transmission":[132],"outputs":[134],"advance,":[139],"which":[140],"subsequently":[141],"serve":[142],"as":[143,163],"exactly":[145],"input":[146,167],"data":[147,168],"corresponding":[152],"downstream":[153],"stages.":[154],"tasks\u2019":[157],"execution":[158],"times":[159],"considerably":[161],"reduced":[162],"most":[164],"their":[166],"already":[170],"been":[171],"local":[173],"nodes":[174],"due":[175],"approach.":[179],"This":[180],"overall":[184],"enhancement":[185],"applications.":[191],"Our":[192,214],"makes":[194],"modifications":[196],"Spark\u2019s":[198],"internal":[199],"working":[200],"mechanism":[201],"achieve":[203],"pre-transmission,":[205],"including":[206],"extensions":[207],"Scheduler":[211],"and":[212],"Executor.":[213],"experiments":[215],"conducted":[216],"utilizing":[217],"HiBench":[219],"benchmark":[220],"demonstrate":[221],"our":[223],"can":[225],"boost":[226],"applications\u2019":[228],"by":[230],"up":[231],"31.3%,":[233],"comparison":[235],"native":[238]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}