{"id":"https://openalex.org/W2516750353","doi":"https://doi.org/10.1145/2949550.2949561","title":"Experiences and Benefits of Running RDMA Hadoop and Spark on SDSC Comet","display_name":"Experiences and Benefits of Running RDMA Hadoop and Spark on SDSC Comet","publication_year":2016,"publication_date":"2016-07-17","ids":{"openalex":"https://openalex.org/W2516750353","doi":"https://doi.org/10.1145/2949550.2949561","mag":"2516750353"},"language":"en","primary_location":{"id":"doi:10.1145/2949550.2949561","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2949550.2949561","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the XSEDE16 Conference on Diversity, Big Data, and Science at Scale","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069831701","display_name":"Mahidhar Tatineni","orcid":"https://orcid.org/0009-0003-0709-090X"},"institutions":[{"id":"https://openalex.org/I181653535","display_name":"San Diego Supercomputer Center","ror":"https://ror.org/04mg3nk07","country_code":"US","type":"facility","lineage":["https://openalex.org/I181653535","https://openalex.org/I36258959"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mahidhar Tatineni","raw_affiliation_strings":["San Diego Supercomputer Center, University of California San Diego, La Jolla, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"San Diego Supercomputer Center, University of California San Diego, La Jolla, CA, USA","institution_ids":["https://openalex.org/I181653535","https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067748041","display_name":"Xiaoyi Lu","orcid":"https://orcid.org/0000-0001-7581-8905"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaoyi Lu","raw_affiliation_strings":["Dept of Computer Science and Engineering, Ohio State University, Columbus, OH, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dept of Computer Science and Engineering, Ohio State University, Columbus, OH, USA","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110442954","display_name":"Dongju Choi","orcid":"https://orcid.org/0009-0009-6651-5326"},"institutions":[{"id":"https://openalex.org/I181653535","display_name":"San Diego Supercomputer Center","ror":"https://ror.org/04mg3nk07","country_code":"US","type":"facility","lineage":["https://openalex.org/I181653535","https://openalex.org/I36258959"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dongju Choi","raw_affiliation_strings":["San Diego Supercomputer Center, University of California San Diego, La Jolla, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"San Diego Supercomputer Center, University of California San Diego, La Jolla, CA, USA","institution_ids":["https://openalex.org/I181653535","https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Amit Majumdar","orcid":null},"institutions":[{"id":"https://openalex.org/I181653535","display_name":"San Diego Supercomputer Center","ror":"https://ror.org/04mg3nk07","country_code":"US","type":"facility","lineage":["https://openalex.org/I181653535","https://openalex.org/I36258959"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amit Majumdar","raw_affiliation_strings":["San Diego Supercomputer Center, University of California San Diego, La Jolla, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"San Diego Supercomputer Center, University of California San Diego, La Jolla, CA, USA","institution_ids":["https://openalex.org/I181653535","https://openalex.org/I36258959"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024879682","display_name":"Dhabaleswar K. Panda","orcid":"https://orcid.org/0000-0002-0356-1781"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dhabaleswar K. (DK) Panda","raw_affiliation_strings":["Dept of Computer Science and Engineering, Ohio State University, Columbus, OH, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dept of Computer Science and Engineering, Ohio State University, Columbus, OH, USA","institution_ids":["https://openalex.org/I52357470"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.505,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.55248344,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8648658990859985},{"id":"https://openalex.org/keywords/remote-direct-memory-access","display_name":"Remote direct memory access","score":0.8034093379974365},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.7145185470581055},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6928756237030029},{"id":"https://openalex.org/keywords/comet","display_name":"Comet","score":0.6894375085830688},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.6574510335922241},{"id":"https://openalex.org/keywords/supercomputer","display_name":"Supercomputer","score":0.5977700352668762},{"id":"https://openalex.org/keywords/middleware","display_name":"Middleware (distributed applications)","score":0.5896235704421997},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.4478324055671692},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.42263656854629517},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.42151832580566406}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8648658990859985},{"id":"https://openalex.org/C130795937","wikidata":"https://www.wikidata.org/wiki/Q2561570","display_name":"Remote direct memory access","level":2,"score":0.8034093379974365},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.7145185470581055},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6928756237030029},{"id":"https://openalex.org/C94081185","wikidata":"https://www.wikidata.org/wiki/Q3559","display_name":"Comet","level":2,"score":0.6894375085830688},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.6574510335922241},{"id":"https://openalex.org/C83283714","wikidata":"https://www.wikidata.org/wiki/Q121117","display_name":"Supercomputer","level":2,"score":0.5977700352668762},{"id":"https://openalex.org/C169468491","wikidata":"https://www.wikidata.org/wiki/Q146923","display_name":"Middleware (distributed applications)","level":2,"score":0.5896235704421997},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.4478324055671692},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.42263656854629517},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.42151832580566406},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C44870925","wikidata":"https://www.wikidata.org/wiki/Q37547","display_name":"Astrophysics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2949550.2949561","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2949550.2949561","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the XSEDE16 Conference on Diversity, Big Data, and Science at Scale","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"},{"id":"https://openalex.org/F4320317220","display_name":"National Energy Research Scientific Computing Center","ror":"https://ror.org/05v3mvq14"},{"id":"https://openalex.org/F4320332603","display_name":"University of California, San Diego","ror":"https://ror.org/0168r3w48"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W1502383304","https://openalex.org/W1581677478","https://openalex.org/W1868798185","https://openalex.org/W1968781019","https://openalex.org/W1984282814","https://openalex.org/W1990082882","https://openalex.org/W2059628440","https://openalex.org/W2068829602","https://openalex.org/W2071704971","https://openalex.org/W2082047517","https://openalex.org/W2123893563"],"related_works":["https://openalex.org/W3090586438","https://openalex.org/W2782433361","https://openalex.org/W2416075414","https://openalex.org/W2475302168","https://openalex.org/W4366999533","https://openalex.org/W2134569538","https://openalex.org/W3198746232","https://openalex.org/W4365450377","https://openalex.org/W4245217724","https://openalex.org/W2907568399"],"abstract_inverted_index":{"Big":[0,153],"Data":[1,154],"problems":[2],"dealing":[3],"with":[4,93,107,113],"a":[5,15,105,160],"variety":[6],"of":[7,18,67,110,130,152,183],"large":[8,97,108],"data":[9,41,60],"sets":[10],"are":[11],"now":[12],"common":[13],"in":[14,189],"wide":[16],"range":[17],"domain":[19],"science":[20],"research":[21],"areas":[22],"such":[23,104],"as":[24,76],"bioinformatics,":[25],"social":[26],"science,":[27],"astronomical":[28],"image":[29],"processing,":[30],"weather":[31],"and":[32,35,43,62,91,99,119,128,139,156],"climate":[33],"dynamics,":[34],"economics.":[36],"In":[37],"some":[38,149,166],"cases,":[39],"the":[40,68,84,143,172,177,184],"generation":[42],"computation":[44],"is":[45,159,168,186],"done":[46],"on":[47,142],"high":[48,89,120],"performance":[49,90,121,150],"computing":[50,73],"(HPC)":[51],"resources,":[52],"thus":[53],"presenting":[54],"an":[55],"incentive":[56],"for":[57,88],"developing/optimizing":[58],"big":[59],"middleware":[61,74,141,173],"tools":[63],"to":[64,170,174],"take":[65],"advantage":[66],"existing":[69],"HPC":[70,146,163,178],"infrastructures.":[71],"Data-intensive":[72],"(such":[75],"Hadoop,":[77],"Spark)":[78],"can":[79],"potentially":[80],"benefit":[81],"greatly":[82],"from":[83],"hardware":[85],"already":[86],"designed":[87],"scalability":[92],"advanced":[94],"processor":[95],"technology,":[96],"memory/core,":[98],"storage/filesystems.":[100],"SDSC":[101],"Comet":[102,145,158],"represents":[103],"resource":[106,164],"numbers":[109],"compute":[111],"nodes":[112],"fast":[114],"node":[115],"local":[116],"SSD":[117],"storage,":[118],"Lustre":[122],"filesystems.":[123],"This":[124,181],"paper":[125],"discusses":[126],"experiences":[127],"benefits":[129],"using":[131],"optimized":[132],"Remote":[133],"Direct":[134],"Memory":[135],"Access":[136],"(RDMA)":[137],"Hadoop":[138],"Spark":[140],"XSEDE":[144],"resource,":[147],"including":[148],"results":[151],"benchmarks":[155],"applications.":[157],"general":[161],"purpose":[162],"so":[165],"work":[167],"needed":[169],"integrate":[171],"run":[175],"within":[176],"scheduling":[179],"framework.":[180],"aspect":[182],"implementation":[185],"also":[187],"discussed":[188],"detail.":[190]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2018,"cited_by_count":2}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}