{"id":"https://openalex.org/W4406460393","doi":"https://doi.org/10.1109/bigdata62323.2024.10825085","title":"Data-Prep-Kit: getting your data ready for LLM application development","display_name":"Data-Prep-Kit: getting your data ready for LLM application development","publication_year":2024,"publication_date":"2024-12-15","ids":{"openalex":"https://openalex.org/W4406460393","doi":"https://doi.org/10.1109/bigdata62323.2024.10825085"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata62323.2024.10825085","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10825085","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000602110","display_name":"David Wood","orcid":"https://orcid.org/0000-0002-1001-2019"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"David Wood","raw_affiliation_strings":["IBM Research,New York,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,New York,USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065545752","display_name":"Boris Lublinsky","orcid":null},"institutions":[{"id":"https://openalex.org/I4210145784","display_name":"IBM Research - Ireland","ror":"https://ror.org/04jnxr720","country_code":"IE","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210145784"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Boris Lublinsky","raw_affiliation_strings":["IBM Research,Mulhaddart,Ireland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,Mulhaddart,Ireland","institution_ids":["https://openalex.org/I4210145784"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114418487","display_name":"Alexy Roytman","orcid":null},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Alexy Roytman","raw_affiliation_strings":["IBM Research Israel,Haifa,Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research Israel,Haifa,Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109715004","display_name":"Shivdeep Singh","orcid":null},"institutions":[{"id":"https://openalex.org/I4210103279","display_name":"IBM Research - India","ror":"https://ror.org/014wt7r80","country_code":"IN","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210103279","https://openalex.org/I4210114115"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Shivdeep Singh","raw_affiliation_strings":["IBM Research India,Bangalore,India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research India,Bangalore,India","institution_ids":["https://openalex.org/I4210103279"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008609792","display_name":"Constantin Adam","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Constantin Adam","raw_affiliation_strings":["IBM Research,New York,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,New York,USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112619087","display_name":"Abdulhamid Adebayo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abdulhamid Adebayo","raw_affiliation_strings":["IBM Research,New York,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,New York,USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103430925","display_name":"Sungeun An","orcid":null},"institutions":[{"id":"https://openalex.org/I4210085935","display_name":"IBM Research - Almaden","ror":"https://ror.org/005w8dd04","country_code":"US","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210085935","https://openalex.org/I4210114115"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sungeun An","raw_affiliation_strings":["Almaden Research Center IBM,San Jose,CA,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Almaden Research Center IBM,San Jose,CA,USA","institution_ids":["https://openalex.org/I4210085935"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087571877","display_name":"Yuan Chang","orcid":"https://orcid.org/0000-0003-1125-4041"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan Chi Chang","raw_affiliation_strings":["IBM Research,New York,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,New York,USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070974608","display_name":"Xuan-Hong Dang","orcid":"https://orcid.org/0000-0003-0018-9109"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xuan-Hong Dang","raw_affiliation_strings":["IBM Research,New York,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,New York,USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113647041","display_name":"Nirmit Desai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nirmit Desai","raw_affiliation_strings":["IBM Research,New York,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,New York,USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114418484","display_name":"Michele Dolfi","orcid":null},"institutions":[{"id":"https://openalex.org/I4210126328","display_name":"IBM Research - Zurich","ror":"https://ror.org/02js37d36","country_code":"CH","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210126328"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Michele Dolfi","raw_affiliation_strings":["IBM Research,Zurich,Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,Zurich,Switzerland","institution_ids":["https://openalex.org/I4210126328"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114418486","display_name":"Hajar Emami-Gohari","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hajar Emami-Gohari","raw_affiliation_strings":["IBM Research,New York,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,New York,USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114418488","display_name":"Revital Eres","orcid":null},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Revital Eres","raw_affiliation_strings":["IBM Research Israel,Haifa,Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research Israel,Haifa,Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Takuya Goto","orcid":null},"institutions":[{"id":"https://openalex.org/I4210145865","display_name":"IBM Research - Tokyo","ror":"https://ror.org/04915qk43","country_code":"JP","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210145865"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Takuya Goto","raw_affiliation_strings":["IBM Software,Tokyo,Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Software,Tokyo,Japan","institution_ids":["https://openalex.org/I4210145865"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103960169","display_name":"Dhiraj Joshi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dhiraj Joshi","raw_affiliation_strings":["IBM Research,New York,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,New York,USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031751148","display_name":"Yan Koyfman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yan Koyfman","raw_affiliation_strings":["IBM Research,New York,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,New York,USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114418489","display_name":"Mohammad Nassar","orcid":null},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Mohammad Nassar","raw_affiliation_strings":["IBM Research Israel,Haifa,Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research Israel,Haifa,Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102917997","display_name":"Hima Patel","orcid":"https://orcid.org/0000-0002-0031-8428"},"institutions":[{"id":"https://openalex.org/I4210103279","display_name":"IBM Research - India","ror":"https://ror.org/014wt7r80","country_code":"IN","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210103279","https://openalex.org/I4210114115"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Hima Patel","raw_affiliation_strings":["IBM Research India,Bangalore,India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research India,Bangalore,India","institution_ids":["https://openalex.org/I4210103279"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114418485","display_name":"Paramesvaran Selvam","orcid":null},"institutions":[{"id":"https://openalex.org/I4210103279","display_name":"IBM Research - India","ror":"https://ror.org/014wt7r80","country_code":"IN","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210103279","https://openalex.org/I4210114115"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Paramesvaran Selvam","raw_affiliation_strings":["IBM Research India,Bangalore,India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research India,Bangalore,India","institution_ids":["https://openalex.org/I4210103279"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114159468","display_name":"Yousaf Shah","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yousaf Shah","raw_affiliation_strings":["IBM Research,New York,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,New York,USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108928258","display_name":"Saptha Surendran","orcid":null},"institutions":[{"id":"https://openalex.org/I4210103279","display_name":"IBM Research - India","ror":"https://ror.org/014wt7r80","country_code":"IN","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210103279","https://openalex.org/I4210114115"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Saptha Surendran","raw_affiliation_strings":["IBM Research India,Bangalore,India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research India,Bangalore,India","institution_ids":["https://openalex.org/I4210103279"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114418483","display_name":"Daiki Tsuzuku","orcid":null},"institutions":[{"id":"https://openalex.org/I4210145865","display_name":"IBM Research - Tokyo","ror":"https://ror.org/04915qk43","country_code":"JP","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210145865"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Daiki Tsuzuku","raw_affiliation_strings":["IBM Software,Tokyo,Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Software,Tokyo,Japan","institution_ids":["https://openalex.org/I4210145865"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048986206","display_name":"Petros Zerfos","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Petros Zerfos","raw_affiliation_strings":["IBM Research,New York,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,New York,USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008258052","display_name":"S. Daijavad","orcid":null},"institutions":[{"id":"https://openalex.org/I4210085935","display_name":"IBM Research - Almaden","ror":"https://ror.org/005w8dd04","country_code":"US","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210085935","https://openalex.org/I4210114115"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shahrokh Daijavad","raw_affiliation_strings":["Almaden Research Center IBM,San Jose,CA,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Almaden Research Center IBM,San Jose,CA,USA","institution_ids":["https://openalex.org/I4210085935"]}]}],"institutions":[],"countries_distinct_count":6,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2234","last_page":"2243"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6166003942489624},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3391129970550537}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6166003942489624},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3391129970550537}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata62323.2024.10825085","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10825085","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.44999998807907104,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W2527672088","https://openalex.org/W2542459869","https://openalex.org/W2946119234","https://openalex.org/W2975169180","https://openalex.org/W3177057043","https://openalex.org/W4236102654","https://openalex.org/W4376167329","https://openalex.org/W4379468930","https://openalex.org/W4382024737","https://openalex.org/W4396814788","https://openalex.org/W4399836557","https://openalex.org/W4402671286","https://openalex.org/W4402950921","https://openalex.org/W4406060224","https://openalex.org/W6747481501","https://openalex.org/W6852584927","https://openalex.org/W6852746770","https://openalex.org/W6866771391","https://openalex.org/W6869770556","https://openalex.org/W6872636145","https://openalex.org/W6876144450","https://openalex.org/W6911436233","https://openalex.org/W7038751623"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Data":[0,30],"preparation":[1,27,46,154],"is":[2,35,163],"the":[3,92,153,168,177],"first":[4],"and":[5,23,37,88,130],"a":[6,57,66,76,119,135,139,164],"very":[7,140],"important":[8],"step":[9],"towards":[10],"any":[11],"Large":[12],"Language":[13],"Model":[14],"(LLM)":[15],"development.":[16],"This":[17],"paper":[18],"introduces":[19],"an":[20],"easy-to-use,":[21],"extensible,":[22],"scale-flexible":[24],"open-source":[25],"data":[26,45,55,174],"toolkit":[28],"called":[29],"Prep":[31],"Kit":[32],"(DPK).":[33],"DPK":[34,51,73,104,128,148,162],"architected":[36],"designed":[38],"to":[39,42,47,63,117,138,167,171,175,184],"enable":[40],"users":[41],"scale":[43,62,137],"their":[44,48,180],"needs.":[49],"With":[50],"they":[52,97],"can":[53,98,111],"prepare":[54,173],"on":[56,65],"local":[58],"machine":[59],"or":[60,115,183],"effortlessly":[61],"run":[64],"cluster":[67],"with":[68,75,187],"thousands":[69],"of":[70,82,121,143,155,179],"CPU":[71],"Cores.":[72],"comes":[74],"highly":[77],"scalable,":[78],"yet":[79],"extensible":[80],"set":[81],"modules":[83,110,146],"that":[84],"transform":[85,107],"natural":[86],"language":[87],"code":[89],"data.":[90],"If":[91],"user":[93],"needs":[94],"additional":[95],"transforms,":[96],"be":[99,112],"easily":[100,172],"developed":[101],"using":[102],"extensive":[103],"support":[105],"for":[106,152],"creation.":[108],"These":[109],"used":[113,151],"independently":[114],"pipelined":[116],"perform":[118],"series":[120],"operations.":[122],"In":[123],"this":[124],"paper,":[125],"we":[126],"describe":[127],"architecture":[129],"show":[131],"its":[132],"performance":[133,178],"from":[134,147],"small":[136],"large":[141],"number":[142],"CPUs.":[144],"The":[145],"have":[149],"been":[150],"Granite":[156],"Models":[157],"[1]":[158],"[2].":[159],"We":[160],"believe":[161],"valuable":[165],"contribution":[166],"AI":[169],"community":[170],"enhance":[176],"LLM":[181],"models":[182,186],"fine-tune":[185],"Retrieval-Augmented":[188],"Generation":[189],"(RAG).":[190]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}