{"id":"https://openalex.org/W7215858024","doi":"https://doi.org/10.32996/jcsts.2021.3.1.7","title":"Big Data Analytics Using Hadoop and Spark: Applications, Challenges, and Future Direction","display_name":"Big Data Analytics Using Hadoop and Spark: Applications, Challenges, and Future Direction","publication_year":2021,"publication_date":"2021-02-28","ids":{"openalex":"https://openalex.org/W7215858024","doi":"https://doi.org/10.32996/jcsts.2021.3.1.7"},"language":"en","primary_location":{"id":"pmh:oai:ojs2.al-kindipublisher.com:article/12518","doi":"https://doi.org/10.32996/jcsts.2021.3.1.7","is_oa":true,"landing_page_url":"https://al-kindipublisher.com/index.php/jcsts/article/view/12518","pdf_url":"https://al-kindipublisher.com/index.php/jcsts/article/download/12518/11209","source":{"id":"https://openalex.org/S4210180307","display_name":"Journal of Computer Science and Technology Studies","issn_l":"2709-104X","issn":["2709-104X"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":"https://openalex.org/P4365383729","host_organization_name":"Al-Kindi Center for Research and Development","host_organization_lineage":["https://openalex.org/P4365383729"],"host_organization_lineage_names":["Al-Kindi Center for Research and Development"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2709-104X","raw_type":"info:eu-repo/semantics/article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://al-kindipublisher.com/index.php/jcsts/article/download/12518/11209","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125516995","display_name":"Surya Veera Brahmaji Rao Sunnam","orcid":"https://orcid.org/0009-0003-0557-9103","observed_orcids":["https://orcid.org/0009-0003-0557-9103"]},"institutions":[{"id":"https://openalex.org/I100621029","display_name":"Bank of America","ror":"https://ror.org/006tvg625","country_code":"US","type":"other","lineage":["https://openalex.org/I100621029"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Surya Veera Brahmaji Rao Sunnam","raw_affiliation_strings":["Lead Data Engineer, Bank of America, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Lead Data Engineer, Bank of America, USA","institution_ids":["https://openalex.org/I100621029"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5125516995"],"corresponding_institution_ids":["https://openalex.org/I100621029"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9276999831199646,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9276999831199646,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14280","display_name":"Big Data Technologies and Applications","score":0.06809999793767929,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12451","display_name":"Smart Grid and Power Systems","score":0.0010000000474974513,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/big-data-analytics","display_name":"big data analytics","score":1.0},{"id":"https://openalex.org/keywords/hadoop","display_name":"Hadoop","score":0.9490000009536743},{"id":"https://openalex.org/keywords/apache-spark","display_name":"Apache Spark","score":1.0},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"distributed computing","score":0.7599999904632568},{"id":"https://openalex.org/keywords/mapreduce","display_name":"MapReduce","score":0.7179999947547913},{"id":"https://openalex.org/keywords/hadoop-distributed-file-system","display_name":"HDFS","score":0.9240000247955322},{"id":"https://openalex.org/keywords/distributed-learning","display_name":"distributed learning","score":0.9779999852180481},{"id":"https://openalex.org/keywords/real-time-analytics","display_name":"real-time analytics","score":0.9850000143051147},{"id":"https://openalex.org/keywords/graph-analysis","display_name":"graph analysis","score":0.6970000267028809},{"id":"https://openalex.org/keywords/in-memory-computing","display_name":"in-memory computing","score":0.7979999780654907},{"id":"https://openalex.org/keywords/anomaly-detection","display_name":"anomaly detection","score":0.7870000004768372},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"natural language processing","score":0.8560000061988831},{"id":"https://openalex.org/keywords/scalability","display_name":"scalability","score":0.6970000267028809},{"id":"https://openalex.org/keywords/data-protection","display_name":"data protection","score":0.8420000076293945},{"id":"https://openalex.org/keywords/resource-management","display_name":"resource management","score":0.8330000042915344},{"id":"https://openalex.org/keywords/cloud-native","display_name":"cloud-native","score":0.6050000190734863},{"id":"https://openalex.org/keywords/edge-computing","display_name":"edge computing","score":1.0},{"id":"https://openalex.org/keywords/hardware-acceleration","display_name":"hardware acceleration","score":0.953000009059906},{"id":"https://openalex.org/keywords/resource-optimization","display_name":"resource optimization","score":0.9210000038146973}],"concepts":[{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.8440999984741211},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8047000169754028},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.7567999958992004},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6546000242233276},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.5389000177383423},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.49320000410079956},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.49149999022483826},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.41909998655319214},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.39070001244544983},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.383899986743927},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.364300012588501},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.35910001397132874},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.35359999537467957},{"id":"https://openalex.org/C138827492","wikidata":"https://www.wikidata.org/wiki/Q6661985","display_name":"Data processing","level":2,"score":0.3327000141143799},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.32499998807907104},{"id":"https://openalex.org/C172367668","wikidata":"https://www.wikidata.org/wiki/Q6504956","display_name":"Data visualization","level":3,"score":0.3246999979019165},{"id":"https://openalex.org/C74197172","wikidata":"https://www.wikidata.org/wiki/Q1195339","display_name":"Directed acyclic graph","level":2,"score":0.31540000438690186},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.31279999017715454},{"id":"https://openalex.org/C175801342","wikidata":"https://www.wikidata.org/wiki/Q1988917","display_name":"Data analysis","level":2,"score":0.3091999888420105},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.3019999861717224},{"id":"https://openalex.org/C162307627","wikidata":"https://www.wikidata.org/wiki/Q204833","display_name":"Enhanced Data Rates for GSM Evolution","level":2,"score":0.2996000051498413},{"id":"https://openalex.org/C76831024","wikidata":"https://www.wikidata.org/wiki/Q5227096","display_name":"Data-intensive computing","level":4,"score":0.296099990606308},{"id":"https://openalex.org/C123593499","wikidata":"https://www.wikidata.org/wiki/Q6008583","display_name":"In-Memory Processing","level":5,"score":0.2919999957084656},{"id":"https://openalex.org/C70061542","wikidata":"https://www.wikidata.org/wiki/Q989016","display_name":"Distributed database","level":2,"score":0.2849000096321106},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2833000123500824},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2669000029563904},{"id":"https://openalex.org/C2778456923","wikidata":"https://www.wikidata.org/wiki/Q5337692","display_name":"Edge computing","level":3,"score":0.26429998874664307},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.25940001010894775},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.25859999656677246},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.2563000023365021}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:ojs2.al-kindipublisher.com:article/12518","doi":"https://doi.org/10.32996/jcsts.2021.3.1.7","is_oa":true,"landing_page_url":"https://al-kindipublisher.com/index.php/jcsts/article/view/12518","pdf_url":"https://al-kindipublisher.com/index.php/jcsts/article/download/12518/11209","source":{"id":"https://openalex.org/S4210180307","display_name":"Journal of Computer Science and Technology Studies","issn_l":"2709-104X","issn":["2709-104X"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":"https://openalex.org/P4365383729","host_organization_name":"Al-Kindi Center for Research and Development","host_organization_lineage":["https://openalex.org/P4365383729"],"host_organization_lineage_names":["Al-Kindi Center for Research and Development"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2709-104X","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:oai:ojs2.al-kindipublisher.com:article/12518","doi":"https://doi.org/10.32996/jcsts.2021.3.1.7","is_oa":true,"landing_page_url":"https://al-kindipublisher.com/index.php/jcsts/article/view/12518","pdf_url":"https://al-kindipublisher.com/index.php/jcsts/article/download/12518/11209","source":{"id":"https://openalex.org/S4210180307","display_name":"Journal of Computer Science and Technology Studies","issn_l":"2709-104X","issn":["2709-104X"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":"https://openalex.org/P4365383729","host_organization_name":"Al-Kindi Center for Research and Development","host_organization_lineage":["https://openalex.org/P4365383729"],"host_organization_lineage_names":["Al-Kindi Center for Research and Development"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2709-104X","raw_type":"info:eu-repo/semantics/article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7215858024.pdf","grobid_xml":"https://content.openalex.org/works/W7215858024.grobid-xml"},"referenced_works_count":23,"referenced_works":["https://openalex.org/W316408160","https://openalex.org/W416578099","https://openalex.org/W2530314904","https://openalex.org/W2897677202","https://openalex.org/W2900588685","https://openalex.org/W2944535974","https://openalex.org/W2979056480","https://openalex.org/W2980795771","https://openalex.org/W3005861485","https://openalex.org/W3006945411","https://openalex.org/W3008215430","https://openalex.org/W3041382323","https://openalex.org/W3093424417","https://openalex.org/W3112375919","https://openalex.org/W3113103529","https://openalex.org/W3124475485","https://openalex.org/W3181204626","https://openalex.org/W3188100601","https://openalex.org/W3215419124","https://openalex.org/W4200184226","https://openalex.org/W4313246833","https://openalex.org/W4365402075","https://openalex.org/W4411615711"],"related_works":["https://openalex.org/W3176501131","https://openalex.org/W4283698594","https://openalex.org/W7216199946","https://openalex.org/W4402413185","https://openalex.org/W4411616046","https://openalex.org/W7204756128","https://openalex.org/W4247566972","https://openalex.org/W4417351649","https://openalex.org/W2905295711","https://openalex.org/W4404586581"],"abstract_inverted_index":{"Big":[0,179],"data":[1,32,40,137],"describes":[2],"large":[3],"datasets":[4],"that":[5],"are":[6,116,149],"difficult":[7],"to":[8,11,168],"manage":[9],"owing":[10],"their":[12,111],"diversity,":[13],"size,":[14],"and":[15,22,50,55,64,68,91,105,113,125,141,159,173],"complexity":[16,142],"in":[17,118],"terms":[18],"of":[19,38,60,102,143,175],"storage,":[20],"analysis,":[21],"visualisation":[23],"for":[24,83],"future":[25,147],"operations.":[26],"Artificial":[27],"intelligence":[28],"(AI)":[29],"based":[30],"big":[31],"analytics":[33],"have":[34],"revolutionised":[35],"the":[36,58,103,146,170,176],"processing":[37],"massive":[39],"sets":[41],"by":[42],"leveraging":[43],"distributed":[44,119],"computing":[45],"platforms":[46],"like":[47],"Apache":[48,51],"Spark":[49,76,106],"Hadoop.":[52],"Both":[53],"HDFS":[54],"MapReduce":[56],"leverage":[57],"5Vs":[59],"Hadoop\u2014Volume,":[61],"Velocity,":[62],"Variety,":[63],"Veracity\u2014to":[65],"store":[66],"files":[67],"perform":[69],"batch":[70],"analysis":[71],"on":[72,154],"massive,":[73],"heterogeneous":[74],"datasets.":[75],"expands":[77],"upon":[78],"these":[79],"capabilities":[80],"with":[81,151],"libraries":[82],"graph":[84],"analytics,":[85,123],"streaming,":[86],"machine":[87,120],"learning,":[88,121],"in-memory":[89],"computing,":[90,158],"directed":[92],"acyclic":[93],"graphs":[94],"(DAGs).":[95],"This":[96],"paper":[97],"gives":[98],"a":[99,152],"general":[100],"outline":[101],"Hadoop":[104],"ecosystem,":[107],"as":[108,110,134,162,164],"well":[109,163],"architecture":[112],"how":[114],"they":[115],"used":[117],"real-time":[122],"NLP,":[124],"anomaly":[126],"detection.":[127],"It":[128],"also":[129],"addresses":[130],"such":[131],"critical":[132],"issues":[133],"scalability":[135],"limitations,":[136],"protection,":[138],"resource":[139,166],"administration,":[140],"infrastructures.":[144],"Lastly,":[145],"directions":[148],"discussed":[150],"focus":[153],"cloud-native":[155],"designs,":[156],"edge":[157],"hardware":[160],"acceleration":[161],"intelligent":[165],"optimization":[167],"improve":[169],"performance,":[171],"efficiency,":[172],"flexibility":[174],"next":[177],"generation":[178],"Data":[180],"systems.":[181]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-03T00:00:00"}