{"id":"https://openalex.org/W4284700449","doi":"https://doi.org/10.14778/3551793.3551855","title":"Fine-grained modeling and optimization for intelligent resource management in big data processing","display_name":"Fine-grained modeling and optimization for intelligent resource management in big data processing","publication_year":2022,"publication_date":"2022-07-01","ids":{"openalex":"https://openalex.org/W4284700449","doi":"https://doi.org/10.14778/3551793.3551855"},"language":"en","primary_location":{"id":"doi:10.14778/3551793.3551855","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3551793.3551855","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2207.02026","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057626684","display_name":"Chenghao Lyu","orcid":null},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Chenghao Lyu","raw_affiliation_strings":["University of Massachusetts, Amherst"],"affiliations":[{"raw_affiliation_string":"University of Massachusetts, Amherst","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101543918","display_name":"Qi Fan","orcid":"https://orcid.org/0000-0002-2644-4457"},"institutions":[{"id":"https://openalex.org/I142476485","display_name":"\u00c9cole Polytechnique","ror":"https://ror.org/05hy3tk52","country_code":"FR","type":"education","lineage":["https://openalex.org/I142476485","https://openalex.org/I4210145102"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Qi Fan","raw_affiliation_strings":["Ecole Polytechnique"],"affiliations":[{"raw_affiliation_string":"Ecole Polytechnique","institution_ids":["https://openalex.org/I142476485"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101633968","display_name":"Fei Song","orcid":"https://orcid.org/0000-0002-8825-6494"},"institutions":[{"id":"https://openalex.org/I142476485","display_name":"\u00c9cole Polytechnique","ror":"https://ror.org/05hy3tk52","country_code":"FR","type":"education","lineage":["https://openalex.org/I142476485","https://openalex.org/I4210145102"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Fei Song","raw_affiliation_strings":["Ecole Polytechnique"],"affiliations":[{"raw_affiliation_string":"Ecole Polytechnique","institution_ids":["https://openalex.org/I142476485"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109671208","display_name":"Arnab Sinha","orcid":null},"institutions":[{"id":"https://openalex.org/I142476485","display_name":"\u00c9cole Polytechnique","ror":"https://ror.org/05hy3tk52","country_code":"FR","type":"education","lineage":["https://openalex.org/I142476485","https://openalex.org/I4210145102"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Arnab Sinha","raw_affiliation_strings":["Ecole Polytechnique"],"affiliations":[{"raw_affiliation_string":"Ecole Polytechnique","institution_ids":["https://openalex.org/I142476485"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109548832","display_name":"Yanlei Diao","orcid":"https://orcid.org/0009-0000-7793-9671"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yanlei Diao","raw_affiliation_strings":["University of Massachusetts, Amherst and Ecole Polytechnique"],"affiliations":[{"raw_affiliation_string":"University of Massachusetts, Amherst and Ecole Polytechnique","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100344511","display_name":"Wei Chen","orcid":"https://orcid.org/0000-0002-9402-6249"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wei Chen","raw_affiliation_strings":["Alibaba Group"],"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115602374","display_name":"Li Ma","orcid":"https://orcid.org/0000-0001-8455-6708"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li Ma","raw_affiliation_strings":["Alibaba Group"],"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111387926","display_name":"Yihui Feng","orcid":"https://orcid.org/0009-0003-7853-9508"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yihui Feng","raw_affiliation_strings":["Alibaba Group"],"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046576694","display_name":"Yaliang Li","orcid":"https://orcid.org/0000-0002-4204-6096"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yaliang Li","raw_affiliation_strings":["Alibaba Group"],"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032503782","display_name":"Kai Zeng","orcid":"https://orcid.org/0009-0005-5788-5668"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kai Zeng","raw_affiliation_strings":["Alibaba Group"],"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5113621558","display_name":"Jingren Zhou","orcid":null},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jingren Zhou","raw_affiliation_strings":["Alibaba Group"],"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":11,"corresponding_author_ids":["https://openalex.org/A5057626684"],"corresponding_institution_ids":["https://openalex.org/I24603500"],"apc_list":null,"apc_paid":null,"fwci":3.1886,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.92945682,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"15","issue":"11","first_page":"3098","last_page":"3111"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9954000115394592,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.819471001625061},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.7575781345367432},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.6102935671806335},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6046729683876038},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.5562842488288879},{"id":"https://openalex.org/keywords/partition","display_name":"Partition (number theory)","score":0.5553344488143921},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.49133363366127014},{"id":"https://openalex.org/keywords/resource-allocation","display_name":"Resource allocation","score":0.477655291557312},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.41236215829849243},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.344074010848999},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.16375067830085754},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.09594452381134033}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.819471001625061},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.7575781345367432},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.6102935671806335},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6046729683876038},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5562842488288879},{"id":"https://openalex.org/C42812","wikidata":"https://www.wikidata.org/wiki/Q1082910","display_name":"Partition (number theory)","level":2,"score":0.5553344488143921},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.49133363366127014},{"id":"https://openalex.org/C29202148","wikidata":"https://www.wikidata.org/wiki/Q287260","display_name":"Resource allocation","level":2,"score":0.477655291557312},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.41236215829849243},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.344074010848999},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.16375067830085754},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.09594452381134033},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.14778/3551793.3551855","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3551793.3551855","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2207.02026","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2207.02026","pdf_url":"https://arxiv.org/pdf/2207.02026","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-03897397v1","is_oa":true,"landing_page_url":"https://inria.hal.science/hal-03897397","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"VLDB 2022 - 48th International Conference on Very Large Databases, Sep 2022, Sydney, Australia","raw_type":"Conference papers"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2207.02026","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2207.02026","pdf_url":"https://arxiv.org/pdf/2207.02026","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.49000000953674316}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":80,"referenced_works":["https://openalex.org/W135863099","https://openalex.org/W1587011656","https://openalex.org/W1789812421","https://openalex.org/W1997188340","https://openalex.org/W2035176107","https://openalex.org/W2039652440","https://openalex.org/W2053688343","https://openalex.org/W2072620509","https://openalex.org/W2082171780","https://openalex.org/W2083541728","https://openalex.org/W2090492562","https://openalex.org/W2105947650","https://openalex.org/W2110086534","https://openalex.org/W2119558533","https://openalex.org/W2131975293","https://openalex.org/W2139072600","https://openalex.org/W2142031898","https://openalex.org/W2151251992","https://openalex.org/W2188062232","https://openalex.org/W2215987640","https://openalex.org/W2258038925","https://openalex.org/W2289698129","https://openalex.org/W2521550930","https://openalex.org/W2528415359","https://openalex.org/W2613206411","https://openalex.org/W2726539084","https://openalex.org/W2752885492","https://openalex.org/W2760770811","https://openalex.org/W2788151607","https://openalex.org/W2791195860","https://openalex.org/W2807637204","https://openalex.org/W2911464154","https://openalex.org/W2911540814","https://openalex.org/W2912516925","https://openalex.org/W2915063781","https://openalex.org/W2948513753","https://openalex.org/W2963355447","https://openalex.org/W2970066309","https://openalex.org/W2970148517","https://openalex.org/W2971285141","https://openalex.org/W2979964543","https://openalex.org/W2991530444","https://openalex.org/W2998249308","https://openalex.org/W3030994385","https://openalex.org/W3032560301","https://openalex.org/W3037027022","https://openalex.org/W3045256801","https://openalex.org/W3097225903","https://openalex.org/W3098844916","https://openalex.org/W3099273181","https://openalex.org/W3100094460","https://openalex.org/W3102340269","https://openalex.org/W3102944959","https://openalex.org/W3105457604","https://openalex.org/W3118844182","https://openalex.org/W3124277639","https://openalex.org/W3127018844","https://openalex.org/W3133029391","https://openalex.org/W3173588310","https://openalex.org/W3173622057","https://openalex.org/W3174394491","https://openalex.org/W3174969457","https://openalex.org/W3176434378","https://openalex.org/W3177010880","https://openalex.org/W3183953540","https://openalex.org/W3188064594","https://openalex.org/W3197977787","https://openalex.org/W3198024709","https://openalex.org/W3198307109","https://openalex.org/W3204950481","https://openalex.org/W3208930906","https://openalex.org/W4206064074","https://openalex.org/W4210689632","https://openalex.org/W4246219036","https://openalex.org/W4247465335","https://openalex.org/W4285338532","https://openalex.org/W4289706945","https://openalex.org/W4312847577","https://openalex.org/W6605485969","https://openalex.org/W6966981211"],"related_works":["https://openalex.org/W4390608645","https://openalex.org/W4247566972","https://openalex.org/W4394895745","https://openalex.org/W2960264696","https://openalex.org/W3090563135","https://openalex.org/W2497432351","https://openalex.org/W4206777497","https://openalex.org/W2910064364","https://openalex.org/W4255224757","https://openalex.org/W2499527417"],"abstract_inverted_index":{"Big":[0],"data":[1,70],"processing":[2],"at":[3,153],"the":[4,64,154,159],"production":[5,138],"scale":[6,65],"presents":[7,83],"a":[8,16,37,84,102,109,134],"highly":[9],"complex":[10],"environment":[11],"for":[12,19,79],"resource":[13,51,92],"optimization":[14,58,93,120],"(RO),":[15],"problem":[17,31],"crucial":[18],"meeting":[20],"performance":[21],"goals":[22],"and":[23,50,60,66,98,118,150,162],"budgetary":[24],"constraints":[25,78],"of":[26,39,45,68,111],"analytical":[27],"users.":[28],"The":[29],"RO":[30,107,130,144],"is":[32,61],"challenging":[33],"because":[34],"it":[35],"involves":[36],"set":[38],"decisions":[40,131],"(the":[41],"partition":[42],"count,":[43],"placement":[44],"parallel":[46],"instances":[47],"on":[48],"machines,":[49],"allocation":[52],"to":[53,74,89,126,158],"each":[54],"instance),":[55],"requires":[56],"multi-objective":[57,91],"(MOO),":[59],"compounded":[62],"by":[63],"complexity":[67],"big":[69],"systems":[71],"while":[72,164],"having":[73],"meet":[75],"stringent":[76],"time":[77],"scheduling.":[80],"This":[81],"paper":[82],"MaxCompute":[85],"based":[86],"integrated":[87],"system":[88,145],"support":[90],"via":[94],"fine-grained":[95,115],"instance-level":[96,129],"modeling":[97],"optimization.":[99],"We":[100],"propose":[101],"new":[103,114,143],"architecture":[104],"that":[105,122,141],"breaks":[106],"into":[108],"series":[110],"simpler":[112],"problems,":[113],"predictive":[116],"models,":[117],"novel":[119],"methods":[121],"exploit":[123],"these":[124],"models":[125],"make":[127],"effective":[128],"well":[132],"under":[133],"second.":[135],"Evaluation":[136],"using":[137],"workloads":[139],"shows":[140],"our":[142],"could":[146],"reduce":[147],"37--72%":[148],"latency":[149],"43--78%":[151],"cost":[152],"same":[155],"time,":[156],"compared":[157],"current":[160],"optimizer":[161],"scheduler,":[163],"running":[165],"in":[166],"0.02-0.23s.":[167]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":1}],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2025-10-10T00:00:00"}