{"id":"https://openalex.org/W4408401189","doi":"https://doi.org/10.1109/ictbig64922.2024.10911053","title":"Advanced Techniques in Data Ingestion and Pipelining for Scalable Big Data Platforms: A Comprehensive Review","display_name":"Advanced Techniques in Data Ingestion and Pipelining for Scalable Big Data Platforms: A Comprehensive Review","publication_year":2024,"publication_date":"2024-12-13","ids":{"openalex":"https://openalex.org/W4408401189","doi":"https://doi.org/10.1109/ictbig64922.2024.10911053"},"language":"en","primary_location":{"id":"doi:10.1109/ictbig64922.2024.10911053","doi":"https://doi.org/10.1109/ictbig64922.2024.10911053","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ictbig64922.2024.10911053","pdf_url":null,"source":{"id":"https://openalex.org/S7407088252","display_name":"IEEE International Conference on ICT in Business Industry & Government (ICTBIG)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE 4th International Conference on ICT in Business Industry & Government (ICTBIG)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5116608212","display_name":"Sachin Murarka","orcid":"https://orcid.org/0009-0002-0409-262X","observed_orcids":["https://orcid.org/0009-0002-0409-262X"]},"institutions":[{"id":"https://openalex.org/I4210111553","display_name":"AISECT University","ror":"https://ror.org/020n82m84","country_code":"IN","type":"education","lineage":["https://openalex.org/I4210111553"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Sachin Murarka","raw_affiliation_strings":["Rabindranath Tagore University,Department of Electronics & Communication Engineering,Bhopal,India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rabindranath Tagore University,Department of Electronics & Communication Engineering,Bhopal,India","institution_ids":["https://openalex.org/I4210111553"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084668152","display_name":"Anshuj Jain","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Anshuj Jain","raw_affiliation_strings":["Scope Global University,Department of Electronics & Communication Engineering,Bhopal,India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Scope Global University,Department of Electronics & Communication Engineering,Bhopal,India","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103979396","display_name":"Laxmi Singh","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210111553","display_name":"AISECT University","ror":"https://ror.org/020n82m84","country_code":"IN","type":"education","lineage":["https://openalex.org/I4210111553"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Laxmi Singh","raw_affiliation_strings":["Rabindranath Tagore University,Department of Electronics & Communication Engineering,Bhopal,India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rabindranath Tagore University,Department of Electronics & Communication Engineering,Bhopal,India","institution_ids":["https://openalex.org/I4210111553"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.0131,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.98061438,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14280","display_name":"Big Data Technologies and Applications","score":0.6265000104904175,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T14280","display_name":"Big Data Technologies and Applications","score":0.6265000104904175,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.3253999948501587,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.03020000085234642,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-ingestion","display_name":"data ingestion","score":0.9990000128746033},{"id":"https://openalex.org/keywords/data-pipelining","display_name":"data pipelining","score":0.9800000190734863},{"id":"https://openalex.org/keywords/big-data-platform","display_name":"big data platform","score":0.996999979019165},{"id":"https://openalex.org/keywords/hadoop","display_name":"Hadoop","score":0.9319999814033508},{"id":"https://openalex.org/keywords/hadoop-distributed-file-system","display_name":"HDFS","score":0.9739999771118164},{"id":"https://openalex.org/keywords/mapreduce","display_name":"MapReduce","score":1.0},{"id":"https://openalex.org/keywords/apache-kafka","display_name":"Apache Kafka","score":0.9990000128746033},{"id":"https://openalex.org/keywords/apache-flume","display_name":"Apache Flume","score":1.0},{"id":"https://openalex.org/keywords/real-time-data-streams","display_name":"real-time data streams","score":0.8529999852180481},{"id":"https://openalex.org/keywords/edge-computing","display_name":"edge computing","score":1.0},{"id":"https://openalex.org/keywords/scalability","display_name":"scalability","score":0.6019999980926514},{"id":"https://openalex.org/keywords/distributed-storage","display_name":"distributed storage","score":0.47999998927116394},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"parallel computing","score":0.6240000128746033}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8244515657424927},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.725434422492981},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.6529103517532349},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.42179161310195923},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.38688740134239197},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.17706754803657532},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.170536607503891}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ictbig64922.2024.10911053","doi":"https://doi.org/10.1109/ictbig64922.2024.10911053","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ictbig64922.2024.10911053","pdf_url":null,"source":{"id":"https://openalex.org/S7407088252","display_name":"IEEE International Conference on ICT in Business Industry & Government (ICTBIG)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE 4th International Conference on ICT in Business Industry & Government (ICTBIG)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure","score":0.8970999717712402}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":7,"referenced_works":["https://openalex.org/W3169012056","https://openalex.org/W4285124172","https://openalex.org/W4307899060","https://openalex.org/W4312884991","https://openalex.org/W4322706912","https://openalex.org/W4387517544","https://openalex.org/W4389064733"],"related_works":["https://openalex.org/W4413450158","https://openalex.org/W4399410264","https://openalex.org/W4405076608","https://openalex.org/W6892777090","https://openalex.org/W6930509806","https://openalex.org/W4405854996","https://openalex.org/W6892671435","https://openalex.org/W4416745515","https://openalex.org/W4399451843","https://openalex.org/W4285260815"],"abstract_inverted_index":{"In":[0,224],"the":[1,17,38,52,127,133,202,229],"era":[2],"of":[3,19,40,55,119,130,204,231],"big":[4,205,216],"data,":[5,217],"effective":[6],"data":[7,23,45,49,57,67,111,120,180,188,206,239],"ingestion":[8,35,144,189,240],"and":[9,21,31,43,80,93,103,109,132,148,160,178,190,195,221,234,241,251,258],"pipelining":[10,50],"are":[11],"critical":[12],"for":[13,46,64,136,187],"organizations":[14,214],"to":[15,37,153,211,226],"harness":[16],"power":[18],"vast":[20],"diverse":[22],"sources,":[24],"such":[25,99,141],"as":[26,100,142],"IoT":[27],"devices,":[28],"social":[29],"media,":[30],"transactional":[32],"systems.":[33],"Data":[34],"refers":[36],"process":[39],"systematically":[41],"collecting":[42],"importing":[44],"processing,":[47],"while":[48,249],"ensures":[51],"continuous":[53],"flow":[54],"this":[56],"through":[58],"various":[59],"transformation":[60],"stages,":[61],"preparing":[62],"it":[63],"analysis.":[65],"As":[66],"volumes":[68],"grow,":[69],"platforms":[70,244],"like":[71],"Hadoop,":[72],"with":[73,126],"its":[74],"HDFS":[75],"(Hadoop":[76],"Distributed":[77],"File":[78],"System)":[79],"MapReduce,":[81],"have":[82],"become":[83],"essential":[84],"in":[85,115,171],"managing":[86],"large-scale":[87],"datasets":[88,131],"by":[89,156],"providing":[90],"distributed":[91],"storage":[92],"parallel":[94],"processing":[95],"capabilities.":[96],"Additionally,":[97],"tools":[98],"Apache":[101,104],"Kafka":[102],"Flume":[105],"facilitate":[106],"both":[107],"batch":[108],"real-time":[110,137],"ingestion,":[112],"ensuring":[113],"flexibility":[114],"handling":[116],"different":[117],"types":[118],"streams.":[121],"However,":[122],"challenges":[123,155],"remain,":[124],"especially":[125],"increasing":[128],"complexity":[129],"growing":[134],"need":[135],"processing.":[138,181],"Emerging":[139],"technologies":[140,186],"cloud-native":[143,232],"frameworks,":[145],"edge":[146],"computing,":[147],"AI-enhanced":[149],"pipelines":[150],"offer":[151,245],"solutions":[152,236],"these":[154],"improving":[157],"scalability,":[158],"performance,":[159],"cost":[161],"efficiency.":[162],"Machine":[163],"learning":[164,253],"is":[165,237],"also":[166],"playing":[167],"an":[168],"evolving":[169],"role":[170],"optimizing":[172],"pipeline":[173],"workflows,":[174],"enabling":[175],"more":[176],"intelligent":[177],"adaptive":[179],"This":[182],"paper":[183],"reviews":[184],"key":[185],"pipelining,":[191],"discusses":[192],"their":[193],"limitations,":[194],"explores":[196],"cutting-edge":[197],"trends":[198],"aimed":[199],"at":[200],"enhancing":[201],"efficiency":[203],"platforms.":[207],"These":[208],"advancements":[209],"promise":[210],"reshape":[212],"how":[213],"handle":[215],"offering":[218],"improved":[219],"agility":[220],"operational":[222],"effectiveness.":[223],"addition":[225],"established":[227],"technologies,":[228],"rise":[230],"architectures":[233],"AI-driven":[235],"revolutionizing":[238],"pipelining.":[242],"Cloud":[243],"scalable,":[246],"on-demand":[247],"resources,":[248],"AI":[250],"machine":[252],"enhance":[254],"automation,":[255],"anomaly":[256],"detection,":[257],"predictive":[259],"analytics":[260],"within":[261],"pipelines.":[262]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":3},{"year":2023,"cited_by_count":1}],"updated_date":"2026-10-26T11:51:55.731654","created_date":"2025-03-14T00:00:00"}