{"id":"https://openalex.org/W4415250824","doi":"https://doi.org/10.1109/hpec67600.2025.11196398","title":"Balancing Performance and Productivity: A Comparative Study of Apache Arrow vs. MPI","display_name":"Balancing Performance and Productivity: A Comparative Study of Apache Arrow vs. MPI","publication_year":2025,"publication_date":"2025-09-15","ids":{"openalex":"https://openalex.org/W4415250824","doi":"https://doi.org/10.1109/hpec67600.2025.11196398"},"language":"en","primary_location":{"id":"doi:10.1109/hpec67600.2025.11196398","is_oa":false,"landing_page_url":"https://doi.org/10.1109/hpec67600.2025.11196398","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE High Performance Extreme Computing Conference (HPEC)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113381396","display_name":"Ritvik Prabhu","orcid":"https://orcid.org/0000-0003-2757-9262"},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ritvik Prabhu","raw_affiliation_strings":["Virginia Tech,Dept. of Computer Science"],"affiliations":[{"raw_affiliation_string":"Virginia Tech,Dept. of Computer Science","institution_ids":["https://openalex.org/I859038795"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5058539554","display_name":"Wu-chun Feng","orcid":"https://orcid.org/0000-0002-6015-0727"},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wu-Chun Feng","raw_affiliation_strings":["Virginia Tech,Dept. of Computer Science"],"affiliations":[{"raw_affiliation_string":"Virginia Tech,Dept. of Computer Science","institution_ids":["https://openalex.org/I859038795"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5113381396"],"corresponding_institution_ids":["https://openalex.org/I859038795"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.44155212,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"7"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14239","display_name":"Business, Education, Mathematics Research","score":0.12189999967813492,"subfield":{"id":"https://openalex.org/subfields/1402","display_name":"Accounting"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T14239","display_name":"Business, Education, Mathematics Research","score":0.12189999967813492,"subfield":{"id":"https://openalex.org/subfields/1402","display_name":"Accounting"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7703999876976013},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.5295000076293945},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.5206000208854675},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.4765999913215637},{"id":"https://openalex.org/keywords/supercomputer","display_name":"Supercomputer","score":0.4717000126838684},{"id":"https://openalex.org/keywords/workstation","display_name":"Workstation","score":0.37130001187324524},{"id":"https://openalex.org/keywords/server","display_name":"Server","score":0.3472000062465668},{"id":"https://openalex.org/keywords/data-structure","display_name":"Data structure","score":0.32339999079704285},{"id":"https://openalex.org/keywords/raw-data","display_name":"Raw data","score":0.3059000074863434}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8493000268936157},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7703999876976013},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.5295000076293945},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.5206000208854675},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5130000114440918},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.4765999913215637},{"id":"https://openalex.org/C83283714","wikidata":"https://www.wikidata.org/wiki/Q121117","display_name":"Supercomputer","level":2,"score":0.4717000126838684},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.42890000343322754},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.37290000915527344},{"id":"https://openalex.org/C67953723","wikidata":"https://www.wikidata.org/wiki/Q192525","display_name":"Workstation","level":2,"score":0.37130001187324524},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.3472000062465668},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.32339999079704285},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.3059000074863434},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.305400013923645},{"id":"https://openalex.org/C201410400","wikidata":"https://www.wikidata.org/wiki/Q1064412","display_name":"Multithreading","level":3,"score":0.3012999892234802},{"id":"https://openalex.org/C76831024","wikidata":"https://www.wikidata.org/wiki/Q5227096","display_name":"Data-intensive computing","level":4,"score":0.2992999851703644},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.29670000076293945},{"id":"https://openalex.org/C2780339515","wikidata":"https://www.wikidata.org/wiki/Q3074698","display_name":"Arrow","level":2,"score":0.2915000021457672},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2888999879360199},{"id":"https://openalex.org/C138959212","wikidata":"https://www.wikidata.org/wiki/Q1806783","display_name":"Load balancing (electrical power)","level":3,"score":0.2833999991416931},{"id":"https://openalex.org/C854659","wikidata":"https://www.wikidata.org/wiki/Q1859284","display_name":"Message passing","level":2,"score":0.2775000035762787},{"id":"https://openalex.org/C138827492","wikidata":"https://www.wikidata.org/wiki/Q6661985","display_name":"Data processing","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.2712000012397766},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.2703999876976013},{"id":"https://openalex.org/C70061542","wikidata":"https://www.wikidata.org/wiki/Q989016","display_name":"Distributed database","level":2,"score":0.2700999975204468},{"id":"https://openalex.org/C150495011","wikidata":"https://www.wikidata.org/wiki/Q128392","display_name":"Concurrent computing","level":2,"score":0.26809999346733093},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.26600000262260437}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/hpec67600.2025.11196398","is_oa":false,"landing_page_url":"https://doi.org/10.1109/hpec67600.2025.11196398","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE High Performance Extreme Computing Conference (HPEC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":8,"referenced_works":["https://openalex.org/W2022658177","https://openalex.org/W2064978316","https://openalex.org/W2090409324","https://openalex.org/W2528015584","https://openalex.org/W4224290816","https://openalex.org/W4235705169","https://openalex.org/W4246367117","https://openalex.org/W4393405311"],"related_works":[],"abstract_inverted_index":{"As":[0],"large-scale":[1,179],"data":[2,50,60],"processing":[3],"becomes":[4],"increasingly":[5],"essential":[6],"in":[7,16,35,71,105,108,112],"today\u2019s":[8],"world,":[9],"balancing":[10,168],"developer":[11,169],"productivity":[12],"and":[13,114,121,127,173],"computational":[14,180],"performance":[15,126],"high-performance":[17],"computing":[18],"(HPC)":[19],"environments":[20,73],"remains":[21],"a":[22,56,81,144],"persistent":[23],"challenge.":[24],"Conventional":[25],"HPC":[26,72],"workloads":[27],"rely":[28],"heavily":[29],"on":[30],"MPI-based":[31],"solutions,":[32],"often":[33],"written":[34],"C":[36],"for":[37,48,100,131,167,178],"the":[38,91,96,123,138],"better":[39],"performance.":[40],"Meanwhile,":[41],"Apache":[42],"Arrow,":[43],"specifically":[44],"leveraging":[45],"Arrow":[46],"Flight":[47],"node-to-node":[49],"transfers,":[51],"has":[52],"gained":[53],"traction":[54],"as":[55,151],"flexible,":[57],"in-memory":[58],"columnar":[59],"approach":[61],"that":[62,84],"promises":[63],"efficient,":[64],"language-agnostic":[65],"memory":[66],"usage,":[67],"although":[68],"its":[69],"usage":[70],"is":[74],"less":[75],"exploredIn":[76],"this":[77,159],"paper,":[78],"we":[79,119,136,152],"perform":[80],"preliminary":[82],"study":[83],"consists":[85],"of":[86,90,129,140,158],"three":[87],"different":[88],"implementations":[89],"same":[92],"data-parallel":[93],"workload":[94],"\u2014":[95,104],"\"Monte":[97],"Carlo":[98],"Simulation":[99],"Financial":[101],"Risk":[102],"Assessment\"":[103],"(1)":[106],"MPI":[107,111],"C,":[109],"(2)":[110],"Python,":[113],"(3)":[115],"Python-based":[116],"Arrow.":[117],"Specifically,":[118],"measure":[120],"compare":[122],"trade-offs":[124],"between":[125],"lines":[128],"code":[130],"each":[132,141],"implementation.":[133],"In":[134],"addition,":[135],"demonstrate":[137],"scalability":[139,174],"implementation":[142],"through":[143],"strong-scaling":[145],"analysis,":[146],"highlighting":[147],"potential":[148],"communication":[149],"bottlenecks":[150],"add":[153],"more":[154],"communicators.":[155],"The":[156],"results":[157],"paper":[160],"are":[161],"intended":[162],"to":[163],"surface":[164],"early":[165],"trends":[166],"productivity,":[170],"raw":[171],"performance,":[172],"when":[175],"selecting":[176],"tools":[177],"tasks.":[181]},"counts_by_year":[],"updated_date":"2026-03-07T16:01:11.037858","created_date":"2025-10-16T00:00:00"}