{"id":"https://openalex.org/W4391093303","doi":"https://doi.org/10.1109/bigdata59044.2023.10386120","title":"MPI4Spark Meets YARN: Enhancing MPI4Spark through YARN support for HPC","display_name":"MPI4Spark Meets YARN: Enhancing MPI4Spark through YARN support for HPC","publication_year":2023,"publication_date":"2023-12-15","ids":{"openalex":"https://openalex.org/W4391093303","doi":"https://doi.org/10.1109/bigdata59044.2023.10386120"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata59044.2023.10386120","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata59044.2023.10386120","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009761772","display_name":"Kinan Al-Attar","orcid":null},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kinan Al-Attar","raw_affiliation_strings":["The Ohio State University,Department of Computer Science Engineering","Department of Computer Science Engineering, The Ohio State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Department of Computer Science Engineering","institution_ids":["https://openalex.org/I52357470"]},{"raw_affiliation_string":"Department of Computer Science Engineering, The Ohio State University","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078128277","display_name":"Aamir Shafi","orcid":"https://orcid.org/0000-0002-1924-2769"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aamir Shafi","raw_affiliation_strings":["The Ohio State University,Department of Computer Science Engineering","Department of Computer Science Engineering, The Ohio State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Department of Computer Science Engineering","institution_ids":["https://openalex.org/I52357470"]},{"raw_affiliation_string":"Department of Computer Science Engineering, The Ohio State University","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034293705","display_name":"Hari Subramoni","orcid":"https://orcid.org/0000-0002-1200-2754"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hari Subramoni","raw_affiliation_strings":["The Ohio State University,Department of Computer Science Engineering","Department of Computer Science Engineering, The Ohio State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Department of Computer Science Engineering","institution_ids":["https://openalex.org/I52357470"]},{"raw_affiliation_string":"Department of Computer Science Engineering, The Ohio State University","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024879682","display_name":"Dhabaleswar K. Panda","orcid":"https://orcid.org/0000-0002-0356-1781"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dhabaleswar K. Panda","raw_affiliation_strings":["The Ohio State University,Department of Computer Science Engineering","Department of Computer Science Engineering, The Ohio State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Department of Computer Science Engineering","institution_ids":["https://openalex.org/I52357470"]},{"raw_affiliation_string":"Department of Computer Science Engineering, The Ohio State University","institution_ids":["https://openalex.org/I52357470"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I52357470"],"apc_list":null,"apc_paid":null,"fwci":1.3974,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.81691261,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"2265","last_page":"2274"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/yarn","display_name":"Yarn","score":0.9743782877922058},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.584221363067627},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.27212825417518616},{"id":"https://openalex.org/keywords/composite-material","display_name":"Composite material","score":0.1512472927570343}],"concepts":[{"id":"https://openalex.org/C2778787235","wikidata":"https://www.wikidata.org/wiki/Q49007","display_name":"Yarn","level":2,"score":0.9743782877922058},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.584221363067627},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.27212825417518616},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.1512472927570343}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata59044.2023.10386120","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata59044.2023.10386120","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":6,"referenced_works":["https://openalex.org/W2107581395","https://openalex.org/W2119738171","https://openalex.org/W2542459869","https://openalex.org/W2584830274","https://openalex.org/W3083442288","https://openalex.org/W4313186662"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2078851640","https://openalex.org/W2381070915","https://openalex.org/W2352481835","https://openalex.org/W2392493391","https://openalex.org/W4200164335","https://openalex.org/W2109471562","https://openalex.org/W2378508949"],"abstract_inverted_index":{"The":[0,96,165,186],"MPI4Spark":[1,129,154,169,191,231,259,288,328],"effort":[2],"was":[3,33,204],"able":[4],"to":[5,107,130,270,315],"reconcile":[6],"disparities":[7],"that":[8,32,155],"existed":[9],"between":[10],"High-Performance":[11],"Computing":[12],"(HPC)":[13],"environments":[14],"and":[15,48,70,111,159,196,212,222,241,250,266,319],"Big":[16],"Data":[17],"stacks,":[18],"by":[19,239,243,264,267,293,333],"adopting":[20],"an":[21,116,179],"MPI-based":[22,180],"solution":[23,133],"inside":[24],"of":[25,35,55,109,140,168,199,225,253,283],"A":[26],"pache":[27],"Spark\u2019s":[28,91],"Netty":[29,181],"communication":[30],"layer":[31,182],"capable":[34],"better":[36,113,261,290],"utilizing":[37],"high-speed":[38],"interconnects":[39],"\u2014":[40,51,142],"such":[41,66],"as":[42,67],"InfiniBand":[43],"(IB),":[44],"Intel":[45,273],"Omni-Path":[46],"(OPA),":[47],"HPE":[49],"Slingshot":[50],"across":[52],"a":[53,132,150,280],"variety":[54],"HPC":[56,117,137,208],"systems.":[57],"Apache":[58],"Spark":[59,195,238,263,292],"provides":[60,112],"support":[61,82,120],"for":[62,102,121,128,136,153,190,295,304,310,317,320],"several":[63],"cluster":[64,76,85,94,98,124,163,223,251,281],"managers,":[65],"YARN,":[68],"Mesos,":[69],"Kubernetes,":[71],"besides":[72],"its":[73,184],"internal":[74,92,161],"standalone":[75,93,162],"manager.":[77,95,164],"MPI4Spark,":[78],"however,":[79],"does":[80],"not":[81],"the":[83,122,160,171,174,296,305],"YARN":[84,97,123,158,176,188,202],"manager,":[86],"instead":[87],"only":[88],"relying":[89],"on":[90,206,277,279,331],"manager":[99,125],"is":[100,126],"designed":[101],"running":[103],"large-scale":[104],"clusters":[105],"up":[106],"hundreds":[108],"nodes":[110],"scalability":[114,141],"in":[115,138,173,233],"environment.":[118],"Therefore,":[119],"needed":[127],"provide":[131],"more":[134],"fitting":[135],"terms":[139],"this":[143,146],"paper":[144],"addresses":[145],"problem.":[147],"We":[148],"present":[149],"new":[151,175,187,201],"design":[152,177,189,203],"supports":[156],"both":[157,193,237],"architectural":[166],"framework":[167],"remains":[170],"same":[172],"with":[178],"at":[183,218],"core.":[185],"outperforms":[192,232],"regular":[194],"RDMA-Spark.":[197,271],"Evaluation":[198],"MPI4Spark\u2019s":[200],"conducted":[205],"two":[207],"systems,":[209],"TACC":[210,213],"Frontera":[211],"Stampede2.":[214],"On":[215,301],"Frontera,":[216,278],"looking":[217],"SortByTest":[219],"weak-scaling":[220],"numbers,":[221,249],"size":[224,252,282],"64":[226],"NodeManagers":[227,255,285],"(3584":[228],"cores,":[229,257],"896GB),":[230],"total":[234,307],"execution":[235,308],"time":[236,309],"4.52x":[240],"RDMA-Spark":[242],"2.33x.":[244],"For":[245,323],"GroupByTest":[246],"strong":[247,324],"scaling":[248,325],"128":[254],"(7168":[256],"1344GB),":[258],"performs":[260],"than":[262,291],"3.29x":[265],"2.32x":[268],"compared":[269,314],"With":[272],"HiBench":[274],"performance":[275],"evaluations":[276],"32":[284],"(1792":[286],"cores),":[287],"fairs":[289],"1.91x":[294],"Logistic":[297],"Regression":[298],"(LR)":[299],"benchmark.":[300],"Stampede2,":[302],"Speed-ups":[303],"overall":[306],"192GB":[311],"are":[312],"1.98x":[313],"IPoIB":[316],"GroupByTest,":[318],"SortByTest,":[321],"1.89x.":[322],"we":[326],"see":[327],"outperforming":[329],"Spark,":[330],"average,":[332],"about":[334],"1.73x":[335],"using":[336],"OHB":[337],"benchmarks.":[338]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3}],"updated_date":"2026-08-12T21:12:35.861297","created_date":"2025-10-10T00:00:00"}