{"id":"https://openalex.org/W2969929817","doi":"https://doi.org/10.1101/741843","title":"ArrowSAM: In-Memory Genomics Data Processing Using Apache Arrow","display_name":"ArrowSAM: In-Memory Genomics Data Processing Using Apache Arrow","publication_year":2019,"publication_date":"2019-08-21","ids":{"openalex":"https://openalex.org/W2969929817","doi":"https://doi.org/10.1101/741843","mag":"2969929817"},"language":"en","primary_location":{"id":"doi:10.1101/741843","doi":"https://doi.org/10.1101/741843","is_oa":true,"landing_page_url":"https://doi.org/10.1101/741843","pdf_url":"https://www.biorxiv.org/content/biorxiv/early/2020/04/06/741843.full.pdf","source":{"id":"https://openalex.org/S4306402567","display_name":"bioRxiv (Cold Spring Harbor Laboratory)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2750212522","host_organization_name":"Cold Spring Harbor Laboratory","host_organization_lineage":["https://openalex.org/I2750212522"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"bioRxiv","raw_type":"posted-content"},"parent_works":[],"lineage":["https://openalex.org/W2969929817"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.biorxiv.org/content/biorxiv/early/2020/04/06/741843.full.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104103554","display_name":"Tanveer Ahmad","orcid":"https://orcid.org/0009-0006-2440-6381","observed_orcids":["https://orcid.org/0009-0006-2440-6381"]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Tanveer Ahmad","raw_affiliation_strings":["Accelerated Big Data Systems, Delft University of Technology, Delft, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Accelerated Big Data Systems, Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101621413","display_name":"Nauman Ahmed","orcid":"https://orcid.org/0000-0002-3964-2348","observed_orcids":["https://orcid.org/0000-0002-3964-2348"]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Nauman Ahmed","raw_affiliation_strings":["Accelerated Big Data Systems, Delft University of Technology, Delft, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Accelerated Big Data Systems, Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029358280","display_name":"Johan Peltenburg","orcid":"https://orcid.org/0000-0002-7043-7131","observed_orcids":["https://orcid.org/0000-0002-7043-7131"]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Johan Peltenburg","raw_affiliation_strings":["Accelerated Big Data Systems, Delft University of Technology, Delft, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Accelerated Big Data Systems, Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021955713","display_name":"Zaid Al-Ars","orcid":"https://orcid.org/0000-0001-7670-8572","observed_orcids":["https://orcid.org/0000-0001-7670-8572"]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Zaid Al-Ars","raw_affiliation_strings":["Accelerated Big Data Systems, Delft University of Technology, Delft, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Accelerated Big Data Systems, Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5104103554"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10015","display_name":"Genomics and Phylogenetic Studies","score":0.5785999894142151,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10015","display_name":"Genomics and Phylogenetic Studies","score":0.5785999894142151,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.20919999480247498,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.08990000188350677,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/apache-arrow","display_name":"Apache Arrow","score":0.7940000295639038},{"id":"https://openalex.org/keywords/in-memory-processing","display_name":"in-memory processing","score":0.7599999904632568},{"id":"https://openalex.org/keywords/genomic-data-processing","display_name":"genomic data processing","score":0.7360000014305115},{"id":"https://openalex.org/keywords/storage-class-memory","display_name":"storage class memory","score":0.9200000166893005},{"id":"https://openalex.org/keywords/cross-language-interoperability","display_name":"cross-language interoperability","score":0.847000002861023},{"id":"https://openalex.org/keywords/bwa-mem","display_name":"BWA-MEM","score":0.9290000200271606},{"id":"https://openalex.org/keywords/picard","display_name":"Picard","score":0.8999999761581421}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8130999803543091},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7267000079154968},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.6146000027656555},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6122000217437744},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.5058000087738037},{"id":"https://openalex.org/C138827492","wikidata":"https://www.wikidata.org/wiki/Q6661985","display_name":"Data processing","level":2,"score":0.48010000586509705},{"id":"https://openalex.org/C189206191","wikidata":"https://www.wikidata.org/wiki/Q222046","display_name":"Genomics","level":4,"score":0.42899999022483826},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.3799999952316284},{"id":"https://openalex.org/C138958017","wikidata":"https://www.wikidata.org/wiki/Q190087","display_name":"Data type","level":2,"score":0.3449999988079071},{"id":"https://openalex.org/C116409475","wikidata":"https://www.wikidata.org/wiki/Q1385056","display_name":"External Data Representation","level":2,"score":0.3206000030040741},{"id":"https://openalex.org/C193702766","wikidata":"https://www.wikidata.org/wiki/Q1414548","display_name":"Concurrency","level":2,"score":0.31119999289512634},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.3111000061035156},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.3091999888420105},{"id":"https://openalex.org/C201797286","wikidata":"https://www.wikidata.org/wiki/Q4914986","display_name":"Biological data","level":2,"score":0.3091999888420105},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3077999949455261},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.2906999886035919},{"id":"https://openalex.org/C75165309","wikidata":"https://www.wikidata.org/wiki/Q2258979","display_name":"Search engine indexing","level":2,"score":0.27799999713897705},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.27230000495910645},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.27219998836517334},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.26579999923706055},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.2612999975681305},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2574000060558319},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.257099986076355},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.2508000135421753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1101/741843","doi":"https://doi.org/10.1101/741843","is_oa":true,"landing_page_url":"https://doi.org/10.1101/741843","pdf_url":"https://www.biorxiv.org/content/biorxiv/early/2020/04/06/741843.full.pdf","source":{"id":"https://openalex.org/S4306402567","display_name":"bioRxiv (Cold Spring Harbor Laboratory)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2750212522","host_organization_name":"Cold Spring Harbor Laboratory","host_organization_lineage":["https://openalex.org/I2750212522"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"bioRxiv","raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.1101/741843","doi":"https://doi.org/10.1101/741843","is_oa":true,"landing_page_url":"https://doi.org/10.1101/741843","pdf_url":"https://www.biorxiv.org/content/biorxiv/early/2020/04/06/741843.full.pdf","source":{"id":"https://openalex.org/S4306402567","display_name":"bioRxiv (Cold Spring Harbor Laboratory)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2750212522","host_organization_name":"Cold Spring Harbor Laboratory","host_organization_lineage":["https://openalex.org/I2750212522"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"bioRxiv","raw_type":"posted-content"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2969929817.pdf","grobid_xml":"https://content.openalex.org/works/W2969929817.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W859230734","https://openalex.org/W1575283894","https://openalex.org/W1583240815","https://openalex.org/W2049185824","https://openalex.org/W2095680943","https://openalex.org/W2108234281","https://openalex.org/W2119406408","https://openalex.org/W2128505445","https://openalex.org/W2132341951","https://openalex.org/W2132378736","https://openalex.org/W2136482257","https://openalex.org/W2152956782","https://openalex.org/W2189371416","https://openalex.org/W2202435095","https://openalex.org/W2371359466","https://openalex.org/W2408045830","https://openalex.org/W2466053952","https://openalex.org/W2526202524","https://openalex.org/W2579241114","https://openalex.org/W2612812586","https://openalex.org/W2745862296","https://openalex.org/W2802080790","https://openalex.org/W2807773783","https://openalex.org/W2909877780","https://openalex.org/W2927075311","https://openalex.org/W2952191974"],"related_works":["https://openalex.org/W3100908593","https://openalex.org/W4313120197","https://openalex.org/W2189371416","https://openalex.org/W2795199554","https://openalex.org/W3013605959","https://openalex.org/W3004467901","https://openalex.org/W1978307270","https://openalex.org/W6948169803","https://openalex.org/W3090472086","https://openalex.org/W4224255422"],"abstract_inverted_index":{"Abstract":[0],"The":[1,180],"rapidly":[2],"growing":[3],"size":[4],"of":[5,55,65,114,138],"genomics":[6,96],"data":[7,23,38,67,82,97,110],"bases,":[8],"driven":[9],"by":[10,126],"advances":[11],"in":[12,28,69,105,131],"sequencing":[13],"technologies,":[14],"demands":[15],"fast":[16,44],"and":[17,32,51,79,117,155,165,170,177,182],"cost-effective":[18],"processing.":[19,45],"However,":[20,71],"processing":[21,80,116],"this":[22,88],"creates":[24],"many":[25],"challenges,":[26],"particularly":[27],"selecting":[29],"appropriate":[30],"algorithms":[31],"computing":[33],"platforms.":[34],"Computing":[35],"systems":[36],"need":[37],"closer":[39,83],"to":[40,48,61,84,112,118,175],"the":[41,85,93,106,136,151],"processor":[42],"for":[43,184],"Traditionally,":[46],"due":[47],"cost,":[49],"volatility":[50],"other":[52],"physical":[53],"constraints":[54],"DRAM,":[56],"it":[57,157],"was":[58],"not":[59],"feasible":[60],"place":[62],"large":[63,128],"amounts":[64],"working":[66],"sets":[68],"memory.":[70],"new":[72],"emerging":[73],"storage":[74],"class":[75],"memories":[76],"allow":[77],"storing":[78],"big":[81],"processor.":[86],"In":[87],"work,":[89],"we":[90,142],"show":[91,168],"how":[92],"commonly":[94],"used":[95],"format,":[98],"Sequence":[99],"Alignment/Map":[100],"(SAM),":[101],"can":[102],"be":[103],"presented":[104],"Apache":[107,152],"Arrow":[108,153],"in-memory":[109,115,146],"representation":[111],"benefit":[113],"ensure":[119],"better":[120],"scalability":[121],"through":[122],"shared":[123],"memory":[124],"objects,":[125],"avoiding":[127],"(de)-serialization":[129],"overheads":[130],"cross-language":[132],"interoperability.":[133],"To":[134],"demonstrate":[135],"benefits":[137],"such":[139],"a":[140],"system,":[141],"propose":[143],"ArrowSAM,":[144],"an":[145],"SAM":[147],"format":[148],"that":[149],"uses":[150],"framework,":[154],"integrate":[156],"into":[158],"genome":[159],"pre-processing":[160],"pipelines":[161],"including":[162],"BWA-MEM,":[163],"Picard":[164,176],"Sambamba.":[166],"Results":[167],"15x":[169],"2.4x":[171],"speedups":[172],"as":[173],"compared":[174],"Sambamba,":[178],"respectively.":[179],"code":[181],"scripts":[183],"running":[185],"all":[186],"workflows":[187],"are":[188],"freely":[189],"available":[190],"at":[191],"https://github.com/abs-tudelft/ArrowSAM":[192],".":[193]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2020,"cited_by_count":2}],"updated_date":"2026-11-14T11:04:58.165337","created_date":"2019-08-29T00:00:00"}