{"id":"https://openalex.org/W4407691326","doi":"https://doi.org/10.1109/tkde.2025.3543225","title":"PipeOptim: Ensuring Effective 1F1B Schedule With Optimizer-Dependent Weight Prediction","display_name":"PipeOptim: Ensuring Effective 1F1B Schedule With Optimizer-Dependent Weight Prediction","publication_year":2025,"publication_date":"2025-02-18","ids":{"openalex":"https://openalex.org/W4407691326","doi":"https://doi.org/10.1109/tkde.2025.3543225"},"language":"en","primary_location":{"id":"doi:10.1109/tkde.2025.3543225","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2025.3543225","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103173849","display_name":"Lei Guan","orcid":"https://orcid.org/0000-0001-9041-3244"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]},{"id":"https://openalex.org/I9086337","display_name":"Taiyuan University of Technology","ror":"https://ror.org/03kv08d37","country_code":"CN","type":"education","lineage":["https://openalex.org/I9086337"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Guan","raw_affiliation_strings":["College of Computer Science and Technology, Taiyuan University of Technology, Taiyuan, China","Department of Mathematics, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0001-9041-3244","affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Taiyuan University of Technology, Taiyuan, China","institution_ids":["https://openalex.org/I9086337"]},{"raw_affiliation_string":"Department of Mathematics, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100440903","display_name":"Dongsheng Li","orcid":"https://orcid.org/0000-0001-9743-2034"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongsheng Li","raw_affiliation_strings":["School of Computer, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0001-9743-2034","affiliations":[{"raw_affiliation_string":"School of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077607914","display_name":"Yongle Chen","orcid":"https://orcid.org/0000-0002-1000-1109"},"institutions":[{"id":"https://openalex.org/I9086337","display_name":"Taiyuan University of Technology","ror":"https://ror.org/03kv08d37","country_code":"CN","type":"education","lineage":["https://openalex.org/I9086337"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongle Chen","raw_affiliation_strings":["College of Computer Science and Technology, Taiyuan University of Technology, Taiyuan, China"],"raw_orcid":"https://orcid.org/0000-0002-1000-1109","affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Taiyuan University of Technology, Taiyuan, China","institution_ids":["https://openalex.org/I9086337"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106626932","display_name":"Jiye Liang","orcid":"https://orcid.org/0000-0001-5887-9327"},"institutions":[{"id":"https://openalex.org/I181877577","display_name":"Shanxi University","ror":"https://ror.org/03y3e3s17","country_code":"CN","type":"education","lineage":["https://openalex.org/I181877577"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiye Liang","raw_affiliation_strings":["School of Computer and Information Technology, Shanxi University, Taiyuan, China"],"raw_orcid":"https://orcid.org/0000-0001-5887-9327","affiliations":[{"raw_affiliation_string":"School of Computer and Information Technology, Shanxi University, Taiyuan, China","institution_ids":["https://openalex.org/I181877577"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100773704","display_name":"Wenjian Wang","orcid":"https://orcid.org/0000-0002-8276-3639"},"institutions":[{"id":"https://openalex.org/I181877577","display_name":"Shanxi University","ror":"https://ror.org/03y3e3s17","country_code":"CN","type":"education","lineage":["https://openalex.org/I181877577"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenjian Wang","raw_affiliation_strings":["School of Computer and Information Technology, Shanxi University, Taiyuan, China"],"raw_orcid":"https://orcid.org/0000-0002-8276-3639","affiliations":[{"raw_affiliation_string":"School of Computer and Information Technology, Shanxi University, Taiyuan, China","institution_ids":["https://openalex.org/I181877577"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102809441","display_name":"Xicheng Lu","orcid":"https://orcid.org/0009-0006-9249-4256"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xicheng Lu","raw_affiliation_strings":["School of Computer, National University of Defense Technology, Changsha, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.792,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.89791213,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":"37","issue":"5","first_page":"2831","last_page":"2845"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.6482999920845032,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.6482999920845032,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7957594394683838},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.518323540687561}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7957594394683838},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.518323540687561},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tkde.2025.3543225","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2025.3543225","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1316732057","display_name":null,"funder_award_id":"62472302","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3606055544","display_name":null,"funder_award_id":"62025208","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7776531264","display_name":null,"funder_award_id":"62476157","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":43,"referenced_works":["https://openalex.org/W1980287119","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2890952074","https://openalex.org/W2962801832","https://openalex.org/W2963917928","https://openalex.org/W2981857663","https://openalex.org/W2991040477","https://openalex.org/W2995059005","https://openalex.org/W3096609285","https://openalex.org/W3129831491","https://openalex.org/W3132977829","https://openalex.org/W3171125843","https://openalex.org/W3206832494","https://openalex.org/W4246193833","https://openalex.org/W4321466422","https://openalex.org/W4378419886","https://openalex.org/W4394711534","https://openalex.org/W4400893425","https://openalex.org/W6631190155","https://openalex.org/W6637373629","https://openalex.org/W6638803421","https://openalex.org/W6676984168","https://openalex.org/W6684191040","https://openalex.org/W6684859321","https://openalex.org/W6727690538","https://openalex.org/W6738975764","https://openalex.org/W6739622702","https://openalex.org/W6743289643","https://openalex.org/W6748425271","https://openalex.org/W6754222578","https://openalex.org/W6755207826","https://openalex.org/W6757817989","https://openalex.org/W6769275798","https://openalex.org/W6769865700","https://openalex.org/W6778883912","https://openalex.org/W6779863968","https://openalex.org/W6780126958","https://openalex.org/W6796414051","https://openalex.org/W6797242466","https://openalex.org/W6799372109","https://openalex.org/W6847028765","https://openalex.org/W6856663864"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Asynchronous":[0],"pipeline":[1,65,135,214],"model":[2],"parallelism":[3],"with":[4],"a":[5,20,77,156],"\u201c1F1B\u201d":[6,25,103,134,165],"(one":[7],"forward,":[8],"one":[9],"backward)":[10],"schedule":[11,26,166],"generates":[12,168],"little":[13],"bubble":[14],"overhead":[15],"and":[16,32,93,167,171,221],"always":[17],"provides":[18],"quite":[19],"high":[21,169],"throughput.":[22],"However,":[23],"the":[24,38,82,98,102,111,117,121,126,133,147,152,161,164,180,183,198,210],"inevitably":[27],"leads":[28],"to":[29,37,85,96,141,150,196],"weight":[30,33,58,78,112,143],"inconsistency":[31],"staleness":[34],"issues":[35],"due":[36],"cross-training":[39],"of":[40,70,101,120,163,179,182,200],"different":[41,193],"mini-batches":[42],"across":[43],"GPUs.":[44],"To":[45,105],"simultaneously":[46],"address":[47],"these":[48],"two":[49],"problems,":[50],"in":[51,81],"this":[52],"paper,":[53],"we":[54,75,108],"propose":[55],"an":[56],"optimizer-dependent":[57],"prediction":[59,79,113],"strategy":[60,80],"(a.k.a":[61],"PipeOptim)":[62],"for":[63],"asynchronous":[64],"training.":[66],"The":[67,203],"key":[68],"insight":[69],"our":[71,201],"proposal":[72],"is":[73,139],"that":[74,88,207],"employ":[76],"forward":[83,99,153],"pass":[84,100],"approximately":[86],"ensure":[87,174],"each":[89,137],"mini-batch":[90,138],"uses":[91],"consistent":[92],"staleness-free":[94],"weights":[95,149],"compute":[97],"schedule.":[104],"be":[106],"concrete,":[107],"first":[109],"construct":[110],"scheme":[114],"based":[115],"on":[116],"update":[118],"rule":[119],"used":[122,184],"optimizer":[123],"when":[124],"training":[125],"deep":[127],"neural":[128],"network":[129],"models.":[130],"Then":[131],"throughout":[132],"training,":[136],"mandated":[140],"execute":[142],"prediction,":[144],"subsequently":[145],"employing":[146],"predicted":[148],"perform":[151],"pass.":[154],"As":[155],"result,":[157],"PipeOptim":[158,208],"1)":[159],"inherits":[160],"advantage":[162],"throughput,":[170],"2)":[172],"can":[173],"effective":[175],"parameter":[176],"learning":[177],"regardless":[178],"type":[181],"optimizer.":[185],"We":[186],"conducted":[187],"extensive":[188],"experimental":[189],"evaluations":[190],"using":[191],"nine":[192],"deep-learning":[194],"models":[195],"verify":[197],"effectiveness":[199],"proposal.":[202],"experiment":[204],"results":[205],"demonstrate":[206],"outperforms":[209],"other":[211],"five":[212],"popular":[213],"approaches":[215],"including":[216],"GPipe,":[217],"PipeDream,":[218],"PipeDream-2BW,":[219],"SpecTrain,":[220],"XPipe.":[222]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2025-12-28T23:10:05.387466","created_date":"2025-10-10T00:00:00"}