{"id":"https://openalex.org/W7154449838","doi":"https://doi.org/10.48550/arxiv.2604.13034","title":"DySkew: Dynamic Data Redistribution for Skew-Resilient Snowpark UDF Execution","display_name":"DySkew: Dynamic Data Redistribution for Skew-Resilient Snowpark UDF Execution","publication_year":2026,"publication_date":"2026-04-14","ids":{"openalex":"https://openalex.org/W7154449838","doi":"https://doi.org/10.48550/arxiv.2604.13034"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.13034","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.13034","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.13034","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133644755","display_name":"Chenwei Xie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xie, Chenwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120604207","display_name":"Urjeet Shrestha","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shrestha, Urjeet","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035212495","display_name":"Corbin McElhanney","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"McElhanney, Corbin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133704237","display_name":"Lukas Lorimer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lorimer, Lukas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133705680","display_name":"Gopal V","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"V, Gopal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133640088","display_name":"Zihao Ye","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ye, Zihao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133696757","display_name":"Yi Pan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pan, Yi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133641024","display_name":"Nic Crouch","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Crouch, Nic","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119892975","display_name":"Elliott Brossard","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Brossard, Elliott","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023481936","display_name":"Florian Funke","orcid":"https://orcid.org/0000-0001-7301-1550"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Funke, Florian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133674644","display_name":"Yuxiong He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Yuxiong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.46959999203681946,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.46959999203681946,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.21690000593662262,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.057500001043081284,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6377999782562256},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.5906999707221985},{"id":"https://openalex.org/keywords/data-migration","display_name":"Data migration","score":0.49410000443458557},{"id":"https://openalex.org/keywords/skew","display_name":"Skew","score":0.4927999973297119},{"id":"https://openalex.org/keywords/dynamic-data","display_name":"Dynamic data","score":0.48969998955726624},{"id":"https://openalex.org/keywords/programming-paradigm","display_name":"Programming paradigm","score":0.4465000033378601},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.4462999999523163},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.41269999742507935},{"id":"https://openalex.org/keywords/fault-tolerance","display_name":"Fault tolerance","score":0.38190001249313354},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.3643999993801117}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8066999912261963},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.7106000185012817},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6377999782562256},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.5906999707221985},{"id":"https://openalex.org/C146152329","wikidata":"https://www.wikidata.org/wiki/Q1932543","display_name":"Data migration","level":2,"score":0.49410000443458557},{"id":"https://openalex.org/C43711488","wikidata":"https://www.wikidata.org/wiki/Q7534783","display_name":"Skew","level":2,"score":0.4927999973297119},{"id":"https://openalex.org/C197298091","wikidata":"https://www.wikidata.org/wiki/Q5318963","display_name":"Dynamic data","level":2,"score":0.48969998955726624},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.4465000033378601},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.4462999999523163},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.41269999742507935},{"id":"https://openalex.org/C63540848","wikidata":"https://www.wikidata.org/wiki/Q3140932","display_name":"Fault tolerance","level":2,"score":0.38190001249313354},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.3643999993801117},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.35920000076293945},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.3450999855995178},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.3431999981403351},{"id":"https://openalex.org/C2989134064","wikidata":"https://www.wikidata.org/wiki/Q288510","display_name":"Execution time","level":2,"score":0.3325999975204468},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.3287000060081482},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.32829999923706055},{"id":"https://openalex.org/C2776834041","wikidata":"https://www.wikidata.org/wiki/Q25346349","display_name":"Execution model","level":2,"score":0.3179999887943268},{"id":"https://openalex.org/C135572916","wikidata":"https://www.wikidata.org/wiki/Q193351","display_name":"Data warehouse","level":2,"score":0.3158999979496002},{"id":"https://openalex.org/C47487241","wikidata":"https://www.wikidata.org/wiki/Q5227230","display_name":"Data access","level":2,"score":0.3100999891757965},{"id":"https://openalex.org/C33762810","wikidata":"https://www.wikidata.org/wiki/Q461671","display_name":"Data integrity","level":2,"score":0.3059000074863434},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.298799991607666},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.2973000109195709},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2971000075340271},{"id":"https://openalex.org/C80445892","wikidata":"https://www.wikidata.org/wiki/Q1052721","display_name":"Multitier architecture","level":3,"score":0.289000004529953},{"id":"https://openalex.org/C97686452","wikidata":"https://www.wikidata.org/wiki/Q7604153","display_name":"Static analysis","level":2,"score":0.2888999879360199},{"id":"https://openalex.org/C88468194","wikidata":"https://www.wikidata.org/wiki/Q1172416","display_name":"Data-flow analysis","level":3,"score":0.28610000014305115},{"id":"https://openalex.org/C107568181","wikidata":"https://www.wikidata.org/wiki/Q5319000","display_name":"Dynamic priority scheduling","level":3,"score":0.2766999900341034},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.2712000012397766},{"id":"https://openalex.org/C2780870223","wikidata":"https://www.wikidata.org/wiki/Q1004415","display_name":"Runtime system","level":2,"score":0.2696000039577484},{"id":"https://openalex.org/C56288433","wikidata":"https://www.wikidata.org/wiki/Q58673","display_name":"Data manipulation language","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C1668388","wikidata":"https://www.wikidata.org/wiki/Q1149776","display_name":"Data management","level":2,"score":0.2632000148296356},{"id":"https://openalex.org/C2781175729","wikidata":"https://www.wikidata.org/wiki/Q7315823","display_name":"Resource distribution","level":3,"score":0.25920000672340393},{"id":"https://openalex.org/C32833848","wikidata":"https://www.wikidata.org/wiki/Q4115054","display_name":"Extensibility","level":2,"score":0.25870001316070557},{"id":"https://openalex.org/C70061542","wikidata":"https://www.wikidata.org/wiki/Q989016","display_name":"Distributed database","level":2,"score":0.25600001215934753},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.2538999915122986},{"id":"https://openalex.org/C2776245389","wikidata":"https://www.wikidata.org/wiki/Q962139","display_name":"Programming style","level":2,"score":0.2522999942302704},{"id":"https://openalex.org/C187191949","wikidata":"https://www.wikidata.org/wiki/Q1138496","display_name":"Profiling (computer programming)","level":2,"score":0.2515000104904175}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.13034","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.13034","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.13034","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.13034","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Snowflake":[0,25],"revolutionized":[1],"data":[2,18,42,71,75,118],"warehousing":[3],"with":[4,133],"an":[5,116,154],"elastic":[6],"architecture":[7,186],"that":[8,40],"decouples":[9],"compute":[10],"and":[11,44,49,80,142,158,190,197,208],"storage,":[12],"enabling":[13],"scalable":[14],"solutions":[15],"for":[16,105,147,167,211],"diverse":[17],"analytics":[19],"needs.":[20],"Building":[21],"on":[22],"this":[23,93,188],"foundation,":[24],"has":[26],"advanced":[27],"its":[28,192],"AI":[29],"Data":[30],"Cloud":[31],"vision":[32],"by":[33],"introducing":[34],"Snowpark,":[35,148],"a":[36,100,159],"managed":[37],"turnkey":[38],"solution":[39],"supports":[41],"engineering":[43],"AI/ML":[45],"workloads":[46],"using":[47],"Python":[48],"other":[50],"programming":[51],"languages.":[52],"While":[53],"Snowpark's":[54],"User-Defined":[55],"Function":[56],"(UDF)":[57],"execution":[58,103,206],"model":[59],"offers":[60],"high":[61],"throughput,":[62],"it":[63],"is":[64],"highly":[65],"vulnerable":[66],"to":[67,163],"performance":[68,195],"degradation":[69],"from":[70],"skew,":[72],"where":[73],"uneven":[74],"partitioning":[76],"causes":[77],"straggler":[78],"tasks":[79],"unpredictable":[81],"latency.":[82],"The":[83],"non-uniform":[84],"computational":[85],"cost":[86],"of":[87,130,135,177,187],"arbitrary":[88],"user":[89],"code":[90],"further":[91],"exacerbates":[92],"classic":[94],"challenge.":[95],"This":[96,171],"paper":[97],"presents":[98],"DySkew,":[99],"novel,":[101],"data-skew-aware":[102],"strategy":[104,157],"Snowpark":[106,213],"UDFs.":[107],"Built":[108],"upon":[109],"Snowflake's":[110],"new":[111],"generalized":[112],"skew":[113],"handling":[114],"solution,":[115],"adaptive":[117],"distribution":[119],"mechanism":[120],"utilizing":[121],"per-link":[122],"state":[123],"machines.":[124],"DySkew":[125],"addresses":[126],"the":[127,175,178,185,205],"unique":[128],"challenges":[129],"user-defined":[131],"logic":[132],"goals":[134],"fine-grained":[136],"per-row":[137],"mitigation,":[138],"dynamic":[139,172],"runtime":[140],"adaptation,":[141],"low-overhead,":[143],"cost-aware":[144],"redistribution.":[145],"Specifically,":[146],"we":[149],"introduce":[150],"crucial":[151],"optimizations,":[152],"including":[153],"eager":[155],"redistribution":[156],"Row":[160],"Size":[161],"Model":[162],"dynamically":[164],"manage":[165],"overhead":[166],"extremely":[168],"large":[169],"rows.":[170],"approach":[173],"replaces":[174],"limitations":[176],"previous":[179],"static":[180],"round-robin":[181],"method.":[182],"We":[183],"detail":[184],"framework":[189],"showcase":[191],"effectiveness":[193],"through":[194],"evaluations":[196],"real-world":[198],"case":[199],"studies,":[200],"demonstrating":[201],"significant":[202],"improvements":[203],"in":[204],"time":[207],"resource":[209],"utilization":[210],"large-scale":[212],"UDF":[214],"workloads.":[215]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-16T00:00:00"}