{"id":"https://openalex.org/W4285173593","doi":"https://doi.org/10.1109/tpds.2022.3187815","title":"AccTFM: An Effective Intra-Layer Model Parallelization Strategy for Training Large-Scale Transformer-Based Models","display_name":"AccTFM: An Effective Intra-Layer Model Parallelization Strategy for Training Large-Scale Transformer-Based Models","publication_year":2022,"publication_date":"2022-07-05","ids":{"openalex":"https://openalex.org/W4285173593","doi":"https://doi.org/10.1109/tpds.2022.3187815"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2022.3187815","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2022.3187815","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043552523","display_name":"Zihao Zeng","orcid":"https://orcid.org/0000-0002-4745-5331"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zihao Zeng","raw_affiliation_strings":["College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-4745-5331","affiliations":[{"raw_affiliation_string":"College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Hunan, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006345319","display_name":"Chubo Liu","orcid":"https://orcid.org/0000-0002-2372-6715"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chubo Liu","raw_affiliation_strings":["College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-2372-6715","affiliations":[{"raw_affiliation_string":"College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Hunan, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008574161","display_name":"Zhuo Tang","orcid":"https://orcid.org/0000-0001-9081-8153"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhuo Tang","raw_affiliation_strings":["College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0001-9081-8153","affiliations":[{"raw_affiliation_string":"College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Hunan, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078793726","display_name":"Kenli Li","orcid":"https://orcid.org/0000-0002-2635-7716"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kenli Li","raw_affiliation_strings":["College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-2635-7716","affiliations":[{"raw_affiliation_string":"College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Hunan, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087894632","display_name":"Keqin Li","orcid":"https://orcid.org/0000-0001-5224-4048"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Keqin Li","raw_affiliation_strings":["College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0001-5224-4048","affiliations":[{"raw_affiliation_string":"College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Hunan, China","institution_ids":["https://openalex.org/I16609230"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I16609230"],"apc_list":null,"apc_paid":null,"fwci":1.0767,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.77642006,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"33","issue":"12","first_page":"4326","last_page":"4338"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8561832308769226},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.5783902406692505},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5778995156288147},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5683351159095764},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5020866394042969},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.4951280653476715},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.4750559329986572},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.45108234882354736},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.4428724944591522},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.4355934262275696},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.34707558155059814},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.19036880135536194},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.13540682196617126}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8561832308769226},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.5783902406692505},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5778995156288147},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5683351159095764},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5020866394042969},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.4951280653476715},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.4750559329986572},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.45108234882354736},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4428724944591522},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.4355934262275696},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.34707558155059814},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.19036880135536194},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.13540682196617126},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2022.3187815","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2022.3187815","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G8971000363","display_name":"\u6df1\u5ea6\u5b66\u4e60\u8ba1\u7b97\u670d\u52a1\u5316\u5f02\u6b65\u5e76\u884c\u5904\u7406\u4f53\u7cfb\u7ed3\u6784\u7814\u7a76","funder_award_id":"62072165","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W24089286","https://openalex.org/W1522734439","https://openalex.org/W1686810756","https://openalex.org/W2064675550","https://openalex.org/W2108598243","https://openalex.org/W2157331557","https://openalex.org/W2163605009","https://openalex.org/W2168231600","https://openalex.org/W2580688187","https://openalex.org/W2606891064","https://openalex.org/W2617960902","https://openalex.org/W2895891814","https://openalex.org/W2896457183","https://openalex.org/W2929084559","https://openalex.org/W2962758826","https://openalex.org/W2963351145","https://openalex.org/W2969388332","https://openalex.org/W2973727699","https://openalex.org/W2975712713","https://openalex.org/W2982475424","https://openalex.org/W2983101505","https://openalex.org/W2985108934","https://openalex.org/W2985738161","https://openalex.org/W2991040477","https://openalex.org/W3047537431","https://openalex.org/W3086105743","https://openalex.org/W3094502228","https://openalex.org/W3101036738","https://openalex.org/W3105668694","https://openalex.org/W3124352525","https://openalex.org/W3137550677","https://openalex.org/W3172612645","https://openalex.org/W3177554119","https://openalex.org/W3211483916","https://openalex.org/W4214612132","https://openalex.org/W4288357791","https://openalex.org/W4292779060","https://openalex.org/W4297822906","https://openalex.org/W4385245566","https://openalex.org/W6600983433","https://openalex.org/W6637373629","https://openalex.org/W6684191040","https://openalex.org/W6684859321","https://openalex.org/W6738250615","https://openalex.org/W6739693220","https://openalex.org/W6739901393","https://openalex.org/W6754818105","https://openalex.org/W6755207826","https://openalex.org/W6756379755","https://openalex.org/W6756718674","https://openalex.org/W6762211661","https://openalex.org/W6767997687","https://openalex.org/W6769275798","https://openalex.org/W6769865700","https://openalex.org/W6774934307","https://openalex.org/W6778883912","https://openalex.org/W6784333009"],"related_works":["https://openalex.org/W1472213334","https://openalex.org/W2124870959","https://openalex.org/W2123996664","https://openalex.org/W94000989","https://openalex.org/W1998323989","https://openalex.org/W1572523360","https://openalex.org/W2887599639","https://openalex.org/W3138436959","https://openalex.org/W2388205394","https://openalex.org/W1513409726"],"abstract_inverted_index":{"Transformer-based":[0],"deep":[1],"neural":[2],"networks":[3],"have":[4],"recently":[5],"swept":[6],"the":[7,89,99,109,115,122,125],"field":[8],"of":[9,64,117,131],"natural":[10],"language":[11],"processing":[12],"due":[13],"to":[14,22,87,113,157,160],"their":[15],"outstanding":[16],"performance,":[17],"and":[18,42,82,102,105,119,137],"are":[19,52],"gradually":[20],"spreading":[21],"more":[23],"applications":[24],"such":[25],"as":[26],"image/video":[27],"processing.":[28],"However,":[29],"compared":[30,159],"with":[31],"general":[32,50],"DNNs,":[33],"training":[34,47,154,163],"a":[35,77],"sizeable":[36],"transformer-based":[37,60,152],"model":[38,70],"is":[39],"further":[40],"time-consuming":[41],"memory-hungry.":[43],"The":[44],"existing":[45],"distributed":[46,162],"strategies":[48],"for":[49,108],"DNNs":[51,153],"not":[53,57],"appropriate":[54],"or":[55],"can":[56],"efficiently":[58],"handle":[59],"networks.":[61],"In":[62],"view":[63],"this,":[65],"we":[66],"propose":[67],"an":[68],"intra-layer":[69],"parallelization":[71],"optimization":[72],"strategy,":[73],"AccTFM,":[74],"which":[75],"introduces":[76],"novel":[78],"fine-grained":[79],"pipeline":[80],"execution":[81],"hybrid":[83,126],"communication":[84,103,127,144],"compression":[85,128],"strategy":[86,112],"overcome":[88],"synchronization":[90],"bottleneck.":[91],"Specifically,":[92],"on":[93],"one":[94],"hand,":[95,124],"it":[96],"first":[97],"decouples":[98],"inter-layer":[100],"computation":[101,118],"dependencies,":[104],"then":[106],"searches":[107],"optimal":[110],"partitioning":[111],"maximize":[114],"overlap":[116],"communication.":[120],"On":[121],"other":[123],"module":[129],"consists":[130],"token-level":[132],"top-":[133],"<inline-formula><tex-math":[134],"notation=\"LaTeX\">$k$</tex-math></inline-formula>":[135],"sparsification":[136],"piecewise":[138],"quantization":[139],"methods":[140],"aiming":[141],"at":[142],"minimizing":[143],"traffic.":[145],"Experimental":[146],"results":[147],"show":[148],"that":[149],"AccTFM":[150],"accelerates":[151],"by":[155],"up":[156],"2.08x":[158],"state-of-the-art":[161],"techniques.":[164]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}