{"id":"https://openalex.org/W7220529263","doi":null,"title":"Optimizing Data Ingestion and Streaming for AI Workloads: A Kafka-Centric Approach","display_name":"Optimizing Data Ingestion and Streaming for AI Workloads: A Kafka-Centric Approach","alternate_titles":[],"publication_year":2022,"publication_date":"2022-12-28","ids":{"openalex":"https://openalex.org/W7220529263"},"language":"en","primary_location":{"id":"pmh:oai:ijmirm.com:article/54","doi":null,"is_oa":true,"landing_page_url":"https://ijmirm.com/index.php/ijmirm/article/view/54","pdf_url":"https://ijmirm.com/index.php/ijmirm/article/download/54/50","source":{"id":"https://openalex.org/S7407056125","display_name":"International Journal of Multidisciplinary Innovation and Research Methodology","issn_l":"2960-2068","issn":["2960-2068"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4404684709","host_organization_name":"Research Radicals","host_organization_lineage":["https://openalex.org/P4404684709"],"host_organization_lineage_names":["Research Radicals"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2960-2068","raw_type":"info:eu-repo/semantics/article"},"parent_works":[],"lineage":["https://openalex.org/W7220529263"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ijmirm.com/index.php/ijmirm/article/download/54/50","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093292513","display_name":"Bhuman Vyas","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"contributor_roles":[],"raw_author_name":"Bhuman Vyas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5093292513"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.2325,"has_fulltext":true,"cited_by_count":20,"citation_normalized_percentile":{"value":0.88333051,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":92,"max":99},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.9707000255584717,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.9707000255584717,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.005400000140070915,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14280","display_name":"Big Data Technologies and Applications","score":0.004699999932199717,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/apache-kafka","display_name":"Apache Kafka","score":1.0},{"id":"https://openalex.org/keywords/data-ingestion","display_name":"data ingestion","score":0.9789999723434448},{"id":"https://openalex.org/keywords/stream-processing","display_name":"stream processing","score":0.8379999995231628},{"id":"https://openalex.org/keywords/ai-workloads","display_name":"AI workloads","score":0.9110000133514404},{"id":"https://openalex.org/keywords/kafka-streams","display_name":"Kafka Streams","score":0.9990000128746033},{"id":"https://openalex.org/keywords/real-time-processing","display_name":"real-time processing","score":0.6050000190734863},{"id":"https://openalex.org/keywords/data-pipelines","display_name":"data pipelines","score":0.7910000085830688},{"id":"https://openalex.org/keywords/partitioning","display_name":"partitioning","score":0.48399999737739563},{"id":"https://openalex.org/keywords/replication","display_name":"replication","score":0.9350000023841858},{"id":"https://openalex.org/keywords/data-connectors","display_name":"data connectors","score":0.3779999911785126},{"id":"https://openalex.org/keywords/batching","display_name":"batching","score":0.5659999847412109},{"id":"https://openalex.org/keywords/serialization","display_name":"data serialization","score":0.6169999837875366},{"id":"https://openalex.org/keywords/data-compression","display_name":"data compression","score":0.7860000133514404},{"id":"https://openalex.org/keywords/data-integrity","display_name":"data integrity","score":0.6129999756813049},{"id":"https://openalex.org/keywords/multi-source-data-fusion","display_name":"multi-source data fusion","score":0.39399999380111694},{"id":"https://openalex.org/keywords/performance-optimization","display_name":"performance optimization","score":0.35199999809265137}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7531999945640564},{"id":"https://openalex.org/C52723943","wikidata":"https://www.wikidata.org/wiki/Q1127410","display_name":"Serialization","level":2,"score":0.6301000118255615},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.4325000047683716},{"id":"https://openalex.org/C89198739","wikidata":"https://www.wikidata.org/wiki/Q3079880","display_name":"Data stream mining","level":2,"score":0.415800005197525},{"id":"https://openalex.org/C107027933","wikidata":"https://www.wikidata.org/wiki/Q2006448","display_name":"Stream processing","level":2,"score":0.3817000091075897},{"id":"https://openalex.org/C2778484313","wikidata":"https://www.wikidata.org/wiki/Q1172540","display_name":"Data stream","level":2,"score":0.34779998660087585},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3361000120639801},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.326200008392334},{"id":"https://openalex.org/C2777611316","wikidata":"https://www.wikidata.org/wiki/Q39045282","display_name":"Streaming data","level":2,"score":0.31029999256134033},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30820000171661377},{"id":"https://openalex.org/C193519340","wikidata":"https://www.wikidata.org/wiki/Q891179","display_name":"Data loss","level":2,"score":0.2987000048160553},{"id":"https://openalex.org/C138827492","wikidata":"https://www.wikidata.org/wiki/Q6661985","display_name":"Data processing","level":2,"score":0.2962000072002411},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.2939999997615814},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.28929999470710754},{"id":"https://openalex.org/C138958017","wikidata":"https://www.wikidata.org/wiki/Q190087","display_name":"Data type","level":2,"score":0.2863999903202057},{"id":"https://openalex.org/C13926793","wikidata":"https://www.wikidata.org/wiki/Q3507155","display_name":"Group method of data handling","level":2,"score":0.2752000093460083},{"id":"https://openalex.org/C2779119184","wikidata":"https://www.wikidata.org/wiki/Q294350","display_name":"ALARM","level":2,"score":0.2632000148296356},{"id":"https://openalex.org/C72634772","wikidata":"https://www.wikidata.org/wiki/Q386824","display_name":"Data integration","level":2,"score":0.2574000060558319},{"id":"https://openalex.org/C33762810","wikidata":"https://www.wikidata.org/wiki/Q461671","display_name":"Data integrity","level":2,"score":0.2533000111579895}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:ijmirm.com:article/54","doi":null,"is_oa":true,"landing_page_url":"https://ijmirm.com/index.php/ijmirm/article/view/54","pdf_url":"https://ijmirm.com/index.php/ijmirm/article/download/54/50","source":{"id":"https://openalex.org/S7407056125","display_name":"International Journal of Multidisciplinary Innovation and Research Methodology","issn_l":"2960-2068","issn":["2960-2068"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4404684709","host_organization_name":"Research Radicals","host_organization_lineage":["https://openalex.org/P4404684709"],"host_organization_lineage_names":["Research Radicals"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2960-2068","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:oai:ijmirm.com:article/54","doi":null,"is_oa":true,"landing_page_url":"https://ijmirm.com/index.php/ijmirm/article/view/54","pdf_url":"https://ijmirm.com/index.php/ijmirm/article/download/54/50","source":{"id":"https://openalex.org/S7407056125","display_name":"International Journal of Multidisciplinary Innovation and Research Methodology","issn_l":"2960-2068","issn":["2960-2068"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4404684709","host_organization_name":"Research Radicals","host_organization_lineage":["https://openalex.org/P4404684709"],"host_organization_lineage_names":["Research Radicals"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2960-2068","raw_type":"info:eu-repo/semantics/article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure","score":0.6773999929428101}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220529263.pdf","grobid_xml":"https://content.openalex.org/works/W7220529263.grobid-xml"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W2241388380","https://openalex.org/W2562924643","https://openalex.org/W2894513539","https://openalex.org/W3033327976","https://openalex.org/W3046567096","https://openalex.org/W3048223833","https://openalex.org/W3197058733","https://openalex.org/W4309675427","https://openalex.org/W4312900800"],"related_works":["https://openalex.org/W6930097749","https://openalex.org/W7052235274","https://openalex.org/W4402186174","https://openalex.org/W6930932695","https://openalex.org/W4408019243","https://openalex.org/W3033327976","https://openalex.org/W4413132066","https://openalex.org/W7108597890","https://openalex.org/W4415268637","https://openalex.org/W4410267391"],"abstract_inverted_index":{"Efficiently":[0],"managing":[1],"data":[2,24,64,73,96,111,119,141,163,191,204,219],"ingestion":[3,65,205],"and":[4,27,49,71,77,88,113,121,131,143,158,170,185,206,217],"streaming":[5,28,36,207],"is":[6],"paramount":[7],"for":[8,29,80,147,182,208,222],"enabling":[9,144],"Artificial":[10],"Intelligence":[11],"(AI)":[12],"workloads":[13],"at":[14,155],"scale.":[15],"This":[16,51,98,193],"paper":[17,99,152,194],"proposes":[18],"a":[19,34],"Kafka-centric":[20],"approach":[21],"to":[22,44,92,118,161],"optimize":[23],"ingestion,":[25],"processing,":[26],"AI":[30,59,148,209,224],"applications.":[31,225],"Apache":[32,200],"Kafka,":[33],"distributed":[35],"platform,":[37],"serves":[38],"as":[39,85],"the":[40,54,101,107,126,151,176,187,196],"backbone":[41],"technology":[42],"due":[43],"its":[45],"robustness,":[46],"fault":[47],"tolerance,":[48],"scalability.":[50],"research":[52],"explores":[53],"integration":[55,108],"of":[56,103,109,128,178,189,199],"Kafka":[57,104,129,132,201],"within":[58],"pipelines,":[60],"focusing":[61],"on":[62],"enhancing":[63,203],"speed,":[66],"ensuring":[67],"real-time":[68],"processing":[69,146],"capabilities,":[70],"maintaining":[72],"integrity.":[74],"Various":[75],"strategies":[76],"best":[78],"practices":[79],"leveraging":[81],"Kafka's":[82,179],"features":[83],"such":[84],"partitions,":[86],"replication,":[87],"connectors":[89],"are":[90],"elucidated":[91],"achieve":[93],"high-throughput,":[94],"low-latency":[95],"streams.":[97],"examines":[100],"role":[102,198],"in":[105,137,202],"facilitating":[106],"diverse":[110],"sources":[112],"formats,":[114,169],"addressing":[115],"challenges":[116],"related":[117],"compatibility":[120],"heterogeneity.":[122],"It":[123,173],"delves":[124],"into":[125,213],"implementation":[127],"Connect":[130],"Streams,":[133],"showcasing":[134],"their":[135],"significance":[136],"seamlessly":[138],"connecting":[139],"disparate":[140],"systems":[142],"stream":[145],"tasks.":[149],"Additionally,":[150],"investigates":[153],"optimizations":[154],"both":[156],"producer":[157],"consumer":[159],"ends":[160],"improve":[162],"throughput,":[164],"including":[165],"batching":[166],"techniques,":[167],"serialization":[168],"compression":[171],"mechanisms.":[172],"also":[174],"discusses":[175],"utilization":[177],"ecosystem":[180],"tools":[181],"monitoring,":[183],"managing,":[184],"optimizing":[186],"performance":[188],"AI-oriented":[190],"pipelines.":[192],"highlights":[195],"pivotal":[197],"workloads,":[210],"offering":[211],"insights":[212],"architecting":[214],"scalable,":[215],"resilient,":[216],"efficient":[218],"pipelines":[220],"essential":[221],"modern":[223]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7220529263","counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":9}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2026-10-06T00:00:00"}
