{"id":"https://openalex.org/W2496453903","doi":"https://doi.org/10.1109/tpds.2016.2591947","title":"A Comprehensive Study of MapReduce Over Lustre for Intermediate Data Placement and Shuffle Strategies on HPC Clusters","display_name":"A Comprehensive Study of MapReduce Over Lustre for Intermediate Data Placement and Shuffle Strategies on HPC Clusters","publication_year":2016,"publication_date":"2016-07-18","ids":{"openalex":"https://openalex.org/W2496453903","doi":"https://doi.org/10.1109/tpds.2016.2591947","mag":"2496453903"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2016.2591947","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2016.2591947","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5112201349","display_name":"Md. Wasi-ur-Rahman","orcid":null},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Md. Wasi-ur-Rahman","raw_affiliation_strings":["Department of Computer Science and Engineering, The Ohio State University, Columbus, OH"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Ohio State University, Columbus, OH","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112525485","display_name":"Nusrat Sharmin Islam","orcid":null},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nusrat Sharmin Islam","raw_affiliation_strings":["Department of Computer Science and Engineering, The Ohio State University, Columbus, OH"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Ohio State University, Columbus, OH","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067748041","display_name":"Xiaoyi Lu","orcid":"https://orcid.org/0000-0001-7581-8905"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaoyi Lu","raw_affiliation_strings":["Department of Computer Science and Engineering, The Ohio State University, Columbus, OH"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Ohio State University, Columbus, OH","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024879682","display_name":"Dhabaleswar K. Panda","orcid":"https://orcid.org/0000-0002-0356-1781"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dhabaleswar K. Panda","raw_affiliation_strings":["Department of Computer Science and Engineering, The Ohio State University, Columbus, OH"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Ohio State University, Columbus, OH","institution_ids":["https://openalex.org/I52357470"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5112201349"],"corresponding_institution_ids":["https://openalex.org/I52357470"],"apc_list":null,"apc_paid":null,"fwci":11.0483,"has_fulltext":false,"cited_by_count":24,"citation_normalized_percentile":{"value":0.98183044,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"28","issue":"3","first_page":"633","last_page":"646"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/lustre","display_name":"Lustre (file system)","score":0.9432535171508789},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8728085160255432},{"id":"https://openalex.org/keywords/remote-direct-memory-access","display_name":"Remote direct memory access","score":0.6375351548194885},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.6317071914672852},{"id":"https://openalex.org/keywords/directory","display_name":"Directory","score":0.561708390712738},{"id":"https://openalex.org/keywords/yarn","display_name":"Yarn","score":0.5566704869270325},{"id":"https://openalex.org/keywords/supercomputer","display_name":"Supercomputer","score":0.5435928106307983},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.5283133387565613},{"id":"https://openalex.org/keywords/data-intensive-computing","display_name":"Data-intensive computing","score":0.4608099162578583},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.4502401649951935},{"id":"https://openalex.org/keywords/file-system","display_name":"File system","score":0.42846179008483887},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.42526674270629883},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.3543868064880371},{"id":"https://openalex.org/keywords/grid-computing","display_name":"Grid computing","score":0.12370967864990234},{"id":"https://openalex.org/keywords/grid","display_name":"Grid","score":0.11322775483131409}],"concepts":[{"id":"https://openalex.org/C180699724","wikidata":"https://www.wikidata.org/wiki/Q1877782","display_name":"Lustre (file system)","level":3,"score":0.9432535171508789},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8728085160255432},{"id":"https://openalex.org/C130795937","wikidata":"https://www.wikidata.org/wiki/Q2561570","display_name":"Remote direct memory access","level":2,"score":0.6375351548194885},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.6317071914672852},{"id":"https://openalex.org/C2777683733","wikidata":"https://www.wikidata.org/wiki/Q201456","display_name":"Directory","level":2,"score":0.561708390712738},{"id":"https://openalex.org/C2778787235","wikidata":"https://www.wikidata.org/wiki/Q49007","display_name":"Yarn","level":2,"score":0.5566704869270325},{"id":"https://openalex.org/C83283714","wikidata":"https://www.wikidata.org/wiki/Q121117","display_name":"Supercomputer","level":2,"score":0.5435928106307983},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.5283133387565613},{"id":"https://openalex.org/C76831024","wikidata":"https://www.wikidata.org/wiki/Q5227096","display_name":"Data-intensive computing","level":4,"score":0.4608099162578583},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4502401649951935},{"id":"https://openalex.org/C2780940931","wikidata":"https://www.wikidata.org/wiki/Q174989","display_name":"File system","level":2,"score":0.42846179008483887},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.42526674270629883},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.3543868064880371},{"id":"https://openalex.org/C70429105","wikidata":"https://www.wikidata.org/wiki/Q249999","display_name":"Grid computing","level":3,"score":0.12370967864990234},{"id":"https://openalex.org/C187691185","wikidata":"https://www.wikidata.org/wiki/Q2020720","display_name":"Grid","level":2,"score":0.11322775483131409},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2016.2591947","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2016.2591947","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5199999809265137,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[{"id":"https://openalex.org/G2946931542","display_name":null,"funder_award_id":"#IIS-1447804","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7363269448","display_name":null,"funder_award_id":"#OCI-1053575","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8472757804","display_name":null,"funder_award_id":"#CNS-1419123","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8481211381","display_name":null,"funder_award_id":"#ACI-1450440","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W300182752","https://openalex.org/W1502383304","https://openalex.org/W1504623286","https://openalex.org/W1504984843","https://openalex.org/W1581677478","https://openalex.org/W1595596071","https://openalex.org/W1963832798","https://openalex.org/W1968781019","https://openalex.org/W1969819821","https://openalex.org/W1990084070","https://openalex.org/W2039373661","https://openalex.org/W2059628440","https://openalex.org/W2067051372","https://openalex.org/W2082047517","https://openalex.org/W2089098642","https://openalex.org/W2105947650","https://openalex.org/W2119738171","https://openalex.org/W2124256298","https://openalex.org/W2173213060","https://openalex.org/W2225892199","https://openalex.org/W4234905039","https://openalex.org/W6630325196","https://openalex.org/W6647746924","https://openalex.org/W6689030548"],"related_works":["https://openalex.org/W300182752","https://openalex.org/W1581677478","https://openalex.org/W2133280212","https://openalex.org/W4311545198","https://openalex.org/W2113098103","https://openalex.org/W2294312748","https://openalex.org/W2140836923","https://openalex.org/W2567604634","https://openalex.org/W2561824179","https://openalex.org/W2770900373"],"abstract_inverted_index":{"With":[0,102],"high":[1],"performance":[2,150],"interconnects":[3],"and":[4,36,63,79,110,199],"parallel":[5],"file":[6],"systems,":[7],"running":[8],"MapReduce":[9,41,75,87,129,175,196],"over":[10,76,173],"modern":[11],"High":[12],"Performance":[13],"Computing":[14],"(HPC)":[15],"clusters":[16,58,90],"has":[17],"attracted":[18],"much":[19],"attention":[20],"due":[21],"to":[22],"its":[23],"uniqueness":[24],"of":[25,33,48,85,184],"solving":[26],"data":[27,116],"analytics":[28],"problems":[29],"with":[30,197],"a":[31,70,81,103,120],"combination":[32],"Big":[34],"Data":[35],"HPC":[37,57,89,157],"technologies.":[38],"Since":[39],"the":[40,46,52,95,132,166,182,189],"architecture":[42,105],"relies":[43],"heavily":[44],"on":[45,73,88],"availability":[47],"local":[49,108],"storage":[50,55,97,135],"media,":[51],"Lustre-based":[53],"global":[54],"in":[56,155],"poses":[59],"many":[60],"new":[61],"opportunities":[62],"challenges.":[64],"In":[65],"this":[66,187],"paper,":[67],"we":[68,118,145],"perform":[69],"comprehensive":[71,192],"study":[72,193],"different":[74],"Lustre":[77,93,111,198],"deployments":[78],"propose":[80,119],"novel":[82,121],"high-performance":[83],"design":[84],"YARN":[86,195],"by":[91,138,170],"utilizing":[92],"as":[94],"additional":[96],"provider":[98],"for":[99,114,152,194],"intermediate":[100,115,134],"data.":[101],"deployment":[104],"where":[106],"both":[107],"disks":[109],"are":[112],"utilized":[113],"storage,":[117],"priority":[122,160],"directory":[123,161],"selection":[124,162],"scheme":[125,163],"through":[126],"which":[127],"RDMA-enhanced":[128],"can":[130,146,164],"choose":[131],"best":[133,183],"during":[136],"runtime":[137],"on-line":[139],"profiling.":[140],"Our":[141,159],"results":[142],"indicate":[143],"that,":[144],"achieve":[147],"44":[148],"percent":[149,172],"benefit":[151],"shuffle-intensive":[153],"workloads":[154],"leadership-class":[156],"systems.":[158],"improve":[165],"job":[167],"execution":[168],"time":[169],"63":[171],"default":[174],"while":[176],"executing":[177],"multiple":[178],"concurrent":[179],"jobs.":[180],"To":[181],"our":[185],"knowledge,":[186],"is":[188],"first":[190],"such":[191],"RDMA.":[200]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":7},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":2}],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-10T00:00:00"}