{"id":"https://openalex.org/W6929438574","doi":"https://doi.org/10.48550/arxiv.2411.11326","title":"Intelligent Pooling: Proactive Resource Provisioning in Large-scale Cloud Service","display_name":"Intelligent Pooling: Proactive Resource Provisioning in Large-scale Cloud Service","publication_year":2024,"publication_date":"2024-11-18","ids":{"openalex":"https://openalex.org/W6929438574","doi":"https://doi.org/10.48550/arxiv.2411.11326"},"language":"en","primary_location":{"id":"doi:10.48550/arxiv.2411.11326","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.11326","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2411.11326","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Ravikumar, Deepak","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Ravikumar, Deepak","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yeo, Alex","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yeo, Alex","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zhu, Yiwen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Yiwen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Lakra, Aditya","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lakra, Aditya","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Nagulapalli, Harsha","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nagulapalli, Harsha","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ravindran, Santhosh Kumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ravindran, Santhosh Kumar","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Suh, Steve","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Suh, Steve","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Dutta, Niharika","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dutta, Niharika","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Fogarty, Andrew","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fogarty, Andrew","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Park, Yoonjae","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Park, Yoonjae","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Khushalani, Sumeet","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Khushalani, Sumeet","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Tarafdar, Arijit","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tarafdar, Arijit","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Parekh, Kunal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Parekh, Kunal","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Krishnan, Subru","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krishnan, Subru","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":14,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10885","display_name":"Gene expression and cancer classification","score":0.5105000138282776,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10885","display_name":"Gene expression and cancer classification","score":0.5105000138282776,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12867","display_name":"Advanced Biosensing Techniques and Applications","score":0.09160000085830688,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11297","display_name":"Ferroptosis and cancer prognosis","score":0.03689999878406525,"subfield":{"id":"https://openalex.org/subfields/2740","display_name":"Pulmonary and Respiratory Medicine"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/provisioning","display_name":"Provisioning","score":0.7717999815940857},{"id":"https://openalex.org/keywords/pooling","display_name":"Pooling","score":0.7476000189781189},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.6449999809265137},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.6420000195503235},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.5218999981880188},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.43149998784065247},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.39980000257492065},{"id":"https://openalex.org/keywords/terabyte","display_name":"Terabyte","score":0.3982999920845032},{"id":"https://openalex.org/keywords/flexibility","display_name":"Flexibility (engineering)","score":0.37130001187324524}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8285999894142151},{"id":"https://openalex.org/C172191483","wikidata":"https://www.wikidata.org/wiki/Q1071806","display_name":"Provisioning","level":2,"score":0.7717999815940857},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.7476000189781189},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.6449999809265137},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.6420000195503235},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.5218999981880188},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.48559999465942383},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.43149998784065247},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.39980000257492065},{"id":"https://openalex.org/C199683683","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Terabyte","level":2,"score":0.3982999920845032},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.37130001187324524},{"id":"https://openalex.org/C29140674","wikidata":"https://www.wikidata.org/wiki/Q206637","display_name":"Computer cluster","level":2,"score":0.364300012588501},{"id":"https://openalex.org/C46637626","wikidata":"https://www.wikidata.org/wiki/Q6693015","display_name":"Low latency (capital markets)","level":2,"score":0.35740000009536743},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.35530000925064087},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.351500004529953},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.34360000491142273},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.325300008058548},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.3239000141620636},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.296999990940094},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2962000072002411},{"id":"https://openalex.org/C164866538","wikidata":"https://www.wikidata.org/wiki/Q367351","display_name":"Cluster (spacecraft)","level":2,"score":0.2935999929904938},{"id":"https://openalex.org/C169468491","wikidata":"https://www.wikidata.org/wiki/Q146923","display_name":"Middleware (distributed applications)","level":2,"score":0.28360000252723694},{"id":"https://openalex.org/C70388272","wikidata":"https://www.wikidata.org/wiki/Q5968558","display_name":"IBM","level":2,"score":0.2799000144004822},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2770000100135803},{"id":"https://openalex.org/C16320812","wikidata":"https://www.wikidata.org/wiki/Q1812200","display_name":"Idle","level":2,"score":0.2687999904155731},{"id":"https://openalex.org/C2778820799","wikidata":"https://www.wikidata.org/wiki/Q3454688","display_name":"Cost reduction","level":2,"score":0.2653999924659729},{"id":"https://openalex.org/C2778160497","wikidata":"https://www.wikidata.org/wiki/Q869830","display_name":"Service-level agreement","level":3,"score":0.2653000056743622},{"id":"https://openalex.org/C19012869","wikidata":"https://www.wikidata.org/wiki/Q578372","display_name":"Response time","level":2,"score":0.2646999955177307}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2411.11326","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.11326","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2411.11326","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.11326","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/8","score":0.6675135493278503,"display_name":"Decent work and economic growth"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0,150],"proliferation":[1],"of":[2,26,32,64,85,207],"big":[3],"data":[4,27],"and":[5,15,43,74,88,132,135,160],"analytic":[6],"workloads":[7],"has":[8],"driven":[9],"the":[10,61,65,82,107,112,138],"need":[11],"for":[12,46,100],"cloud":[13],"compute":[14,103],"cluster-based":[16],"job":[17],"processing.":[18],"With":[19],"Apache":[20],"Spark,":[21],"users":[22,47],"can":[23],"process":[24],"terabytes":[25],"at":[28,39],"ease":[29],"with":[30,129,163],"hundreds":[31],"parallel":[33],"executors.":[34],"At":[35],"Microsoft,":[36],"we":[37,94],"aim":[38],"providing":[40],"a":[41,77,98],"fast":[42],"succinct":[44],"interface":[45],"to":[48,71,81,105,142,156,165,178,186,204,216],"run":[49],"Spark":[50,72],"applications,":[51],"such":[52],"as":[53,214],"through":[54],"creating":[55],"simple":[56],"notebook":[57],"\"sessions\"":[58],"by":[59],"abstracting":[60],"underlying":[62],"complexity":[63],"cloud.":[66],"Providing":[67],"low":[68,130],"latency":[69,131],"access":[70],"clusters":[73],"sessions":[75],"is":[76,201],"challenging":[78],"problem":[79],"due":[80],"large":[83],"overheads":[84],"cluster":[86,182],"creation":[87],"session":[89],"startup.":[90],"In":[91],"this":[92],"paper,":[93],"introduce":[95],"Intelligent":[96,174,199],"Pooling,":[97],"system":[99,116,152],"proactively":[101],"provisioning":[102],"resources":[104],"combat":[106],"aforementioned":[108],"overheads.":[109],"To":[110],"reduce":[111],"COGS":[113,211],"(cost-of-goods-sold),":[114],"our":[115],"(1)":[117],"predicts":[118],"usage":[119],"patterns":[120],"using":[121,170],"an":[122],"innovative":[123],"hybrid":[124],"Machine":[125],"Learning":[126],"(ML)":[127],"model":[128],"high":[133],"accuracy;":[134],"(2)":[136],"optimizes":[137],"pool":[139,192],"size":[140],"dynamically":[141],"meet":[143],"customer":[144],"demand":[145],"while":[146],"reducing":[147],"extraneous":[148],"COGS.":[149],"proposed":[151],"auto-tunes":[153],"its":[154],"hyper-parameters":[155],"balance":[157],"between":[158],"performance":[159],"operational":[161],"cost":[162],"minimal":[164],"no":[166],"engineering":[167],"input.":[168],"Evaluated":[169],"large-scale":[171],"production":[172],"data,":[173],"Pooling":[175,200],"achieves":[176],"up":[177],"43%":[179],"reduction":[180],"in":[181,197,210],"idle":[183],"time":[184],"compared":[185,215],"static":[187],"pooling":[188],"when":[189],"targeting":[190],"99%":[191],"hit":[193],"rate.":[194],"Currently":[195],"deployed":[196],"production,":[198],"on":[202],"track":[203],"save":[205],"tens":[206],"million":[208],"dollars":[209],"per":[212],"year":[213],"traditional":[217],"pre-provisioned":[218],"pools.":[219]},"counts_by_year":[],"updated_date":"2025-11-06T06:51:31.235846","created_date":"2025-10-10T00:00:00"}