{"id":"https://openalex.org/W7134897182","doi":"https://doi.org/10.1109/icdmw69685.2025.00064","title":"Efficient Deadline-Constrained Scheduling and Hybrid Parallelism Optimization of Distributed AutoML Pipelines","display_name":"Efficient Deadline-Constrained Scheduling and Hybrid Parallelism Optimization of Distributed AutoML Pipelines","publication_year":2025,"publication_date":"2025-11-12","ids":{"openalex":"https://openalex.org/W7134897182","doi":"https://doi.org/10.1109/icdmw69685.2025.00064"},"language":null,"primary_location":{"id":"doi:10.1109/icdmw69685.2025.00064","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icdmw69685.2025.00064","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Data Mining Workshops (ICDMW)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061336784","display_name":"Nikolay Butakov","orcid":"https://orcid.org/0000-0002-2705-1313"},"institutions":[{"id":"https://openalex.org/I173089394","display_name":"ITMO University","ror":"https://ror.org/04txgxn49","country_code":"RU","type":"education","lineage":["https://openalex.org/I173089394"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Nikolay Butakov","raw_affiliation_strings":["ITMO University,Industrial AI Research Lab,Saint-Petersburg,Russia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ITMO University,Industrial AI Research Lab,Saint-Petersburg,Russia","institution_ids":["https://openalex.org/I173089394"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051971866","display_name":"Anton Vakhrushev","orcid":null},"institutions":[{"id":"https://openalex.org/I4210116494","display_name":"Timber Institute","ror":"https://ror.org/02phgvp38","country_code":"CZ","type":"other","lineage":["https://openalex.org/I4210116494"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Anton Vakhrushev","raw_affiliation_strings":["Sber AI Lab, Sber,Moscow,Russia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sber AI Lab, Sber,Moscow,Russia","institution_ids":["https://openalex.org/I4210116494"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049619472","display_name":"Maria Khodorchenko","orcid":"https://orcid.org/0000-0001-5446-5311"},"institutions":[{"id":"https://openalex.org/I173089394","display_name":"ITMO University","ror":"https://ror.org/04txgxn49","country_code":"RU","type":"education","lineage":["https://openalex.org/I173089394"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Maria Khodorchenko","raw_affiliation_strings":["ITMO University,Industrial AI Research Lab,Saint-Petersburg,Russia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ITMO University,Industrial AI Research Lab,Saint-Petersburg,Russia","institution_ids":["https://openalex.org/I173089394"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065756178","display_name":"Anastasiia Zakharova","orcid":"https://orcid.org/0000-0002-7624-6790"},"institutions":[{"id":"https://openalex.org/I173089394","display_name":"ITMO University","ror":"https://ror.org/04txgxn49","country_code":"RU","type":"education","lineage":["https://openalex.org/I173089394"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Anastasiia Zakharova","raw_affiliation_strings":["ITMO University,Industrial AI Research Lab,Saint-Petersburg,Russia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ITMO University,Industrial AI Research Lab,Saint-Petersburg,Russia","institution_ids":["https://openalex.org/I173089394"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037894905","display_name":"Oleksandr S. Ryzhkov","orcid":"https://orcid.org/0000-0003-0535-7722"},"institutions":[{"id":"https://openalex.org/I4210116494","display_name":"Timber Institute","ror":"https://ror.org/02phgvp38","country_code":"CZ","type":"other","lineage":["https://openalex.org/I4210116494"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Alexander Ryzhkov","raw_affiliation_strings":["Sber AI Lab, Sber,Moscow,Russia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sber AI Lab, Sber,Moscow,Russia","institution_ids":["https://openalex.org/I4210116494"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128720580","display_name":"Olga Plosskaya","orcid":null},"institutions":[{"id":"https://openalex.org/I4210116494","display_name":"Timber Institute","ror":"https://ror.org/02phgvp38","country_code":"CZ","type":"other","lineage":["https://openalex.org/I4210116494"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Olga Plosskaya","raw_affiliation_strings":["Sber AI Lab, Sber,Moscow,Russia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sber AI Lab, Sber,Moscow,Russia","institution_ids":["https://openalex.org/I4210116494"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059046836","display_name":"Azamat R. Gainetdinov","orcid":null},"institutions":[{"id":"https://openalex.org/I173089394","display_name":"ITMO University","ror":"https://ror.org/04txgxn49","country_code":"RU","type":"education","lineage":["https://openalex.org/I173089394"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Azamat Gainetdinov","raw_affiliation_strings":["ITMO University,Industrial AI Research Lab,Saint-Petersburg,Russia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ITMO University,Industrial AI Research Lab,Saint-Petersburg,Russia","institution_ids":["https://openalex.org/I173089394"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021252788","display_name":"Rinchin Damdinov","orcid":null},"institutions":[{"id":"https://openalex.org/I4210116494","display_name":"Timber Institute","ror":"https://ror.org/02phgvp38","country_code":"CZ","type":"other","lineage":["https://openalex.org/I4210116494"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Rinchin Damdinov","raw_affiliation_strings":["Sber AI Lab, Sber,Moscow,Russia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sber AI Lab, Sber,Moscow,Russia","institution_ids":["https://openalex.org/I4210116494"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019012874","display_name":"Dmitriy Alexandrov","orcid":null},"institutions":[{"id":"https://openalex.org/I173089394","display_name":"ITMO University","ror":"https://ror.org/04txgxn49","country_code":"RU","type":"education","lineage":["https://openalex.org/I173089394"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Dmitriy Alexandrov","raw_affiliation_strings":["ITMO University,Industrial AI Research Lab,Saint-Petersburg,Russia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ITMO University,Industrial AI Research Lab,Saint-Petersburg,Russia","institution_ids":["https://openalex.org/I173089394"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022864448","display_name":"Denis Nasonov","orcid":null},"institutions":[{"id":"https://openalex.org/I173089394","display_name":"ITMO University","ror":"https://ror.org/04txgxn49","country_code":"RU","type":"education","lineage":["https://openalex.org/I173089394"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Denis Nasonov","raw_affiliation_strings":["ITMO University,Industrial AI Research Lab,Saint-Petersburg,Russia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ITMO University,Industrial AI Research Lab,Saint-Petersburg,Russia","institution_ids":["https://openalex.org/I173089394"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028640661","display_name":"Maxim Savchenko","orcid":null},"institutions":[{"id":"https://openalex.org/I4210116494","display_name":"Timber Institute","ror":"https://ror.org/02phgvp38","country_code":"CZ","type":"other","lineage":["https://openalex.org/I4210116494"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Maxim Savchenko","raw_affiliation_strings":["Sber AI Lab, Sber,Moscow,Russia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sber AI Lab, Sber,Moscow,Russia","institution_ids":["https://openalex.org/I4210116494"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":11,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.82733842,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"520","last_page":"528"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.7159000039100647,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.7159000039100647,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11596","display_name":"Constraint Satisfaction and Optimization","score":0.030400000512599945,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.021199999377131462,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6409000158309937},{"id":"https://openalex.org/keywords/pipeline-transport","display_name":"Pipeline transport","score":0.5001999735832214},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.3418999910354614},{"id":"https://openalex.org/keywords/job-shop-scheduling","display_name":"Job shop scheduling","score":0.3246999979019165},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.28040000796318054}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.70169997215271},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6409000158309937},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.5001999735832214},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4625999927520752},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.38119998574256897},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.3418999910354614},{"id":"https://openalex.org/C55416958","wikidata":"https://www.wikidata.org/wiki/Q6206757","display_name":"Job shop scheduling","level":3,"score":0.3246999979019165},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2849000096321106},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.28380000591278076},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.28040000796318054},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.2623000144958496},{"id":"https://openalex.org/C2984822820","wikidata":"https://www.wikidata.org/wiki/Q1123036","display_name":"Processor scheduling","level":3,"score":0.25189998745918274}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icdmw69685.2025.00064","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icdmw69685.2025.00064","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Data Mining Workshops (ICDMW)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W154343146","https://openalex.org/W2100518191","https://openalex.org/W2132862423","https://openalex.org/W2185864411","https://openalex.org/W2306821251","https://openalex.org/W2547190417","https://openalex.org/W2547386789","https://openalex.org/W2809374586","https://openalex.org/W2949676527","https://openalex.org/W2955219525","https://openalex.org/W2962725887","https://openalex.org/W2966284335","https://openalex.org/W3123920941","https://openalex.org/W3129833743","https://openalex.org/W3209503812","https://openalex.org/W4210358484","https://openalex.org/W4210471555","https://openalex.org/W4372349381","https://openalex.org/W4388581326","https://openalex.org/W4390263770","https://openalex.org/W4392309097","https://openalex.org/W4394927283","https://openalex.org/W4401879320","https://openalex.org/W4408120591","https://openalex.org/W4408182811"],"related_works":[],"abstract_inverted_index":{"Updating":[0],"machine":[1],"learning":[2],"(ML)":[3],"models":[4],"in":[5,59],"production":[6,157],"on":[7,113],"a":[8,48],"daily":[9],"basis":[10],"is":[11],"increasingly":[12],"common,":[13],"but":[14,36],"these":[15],"updates":[16,163],"often":[17],"need":[18],"to":[19,72],"meet":[20],"strict":[21],"deadlines.":[22,138],"To":[23],"address":[24],"this,":[25],"automated":[26],"ML":[27],"(AutoML)":[28],"systems":[29,145],"must":[30],"not":[31],"only":[32],"optimize":[33],"model":[34,109,162],"performance":[35],"also":[37],"schedule":[38],"computations":[39],"efficiently":[40],"within":[41],"limited":[42],"time":[43,82,95],"budgets.":[44],"This":[45,139],"paper":[46],"presents":[47],"deadline-aware":[49,150],"optimization":[50],"framework":[51],"for":[52,156],"distributed":[53,143],"AutoML":[54,144],"pipelines,":[55],"leveraging":[56],"hybrid":[57],"parallelism":[58,103],"Apache":[60],"Spark.":[61],"By":[62],"modeling":[63],"both":[64],"data":[65],"and":[66,75,89,108,115,127,151,164],"compute":[67],"parallelism,":[68],"our":[69,121],"approach":[70],"adapts":[71],"varying":[73],"infrastructure":[74],"dataset":[76],"characteristics,":[77],"ensuring":[78],"optimal":[79],"allocation":[80],"of":[81,102],"across":[83],"multiple":[84],"models.":[85],"We":[86],"introduce":[87],"runtime":[88],"scalability":[90],"metamodels":[91],"that":[92,105,120,159],"predict":[93],"execution":[94],"under":[96,136],"different":[97],"configurations,":[98],"guiding":[99],"the":[100],"selection":[101],"strategies":[104],"balance":[106],"efficiency":[107],"accuracy.":[110],"Through":[111],"experiments":[112],"realworld":[114],"synthetic":[116],"datasets,":[117],"we":[118],"demonstrate":[119],"method":[122],"consistently":[123],"outperforms":[124],"traditional":[125],"data-parallel":[126],"computeparallel":[128],"approaches,":[129],"improving":[130],"trial":[131],"completion":[132],"rates":[133],"by":[134],"20-40%":[135],"tight":[137],"work":[140],"highlights":[141],"how":[142],"can":[146],"be":[147],"made":[148],"more":[149],"resourceefficient,":[152],"with":[153],"broad":[154],"implications":[155],"pipelines":[158],"require":[160],"frequent":[161],"real-time":[165],"decision-making.":[166]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-03-12T00:00:00"}