{"id":"https://openalex.org/W4392910546","doi":"https://doi.org/10.1109/icassp48485.2024.10446916","title":"3D Parallelism for Transformers via Integer Programming","display_name":"3D Parallelism for Transformers via Integer Programming","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4392910546","doi":"https://doi.org/10.1109/icassp48485.2024.10446916"},"language":"en","primary_location":{"id":"doi:10.1109/icassp48485.2024.10446916","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10446916","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100785439","display_name":"Hao Zheng","orcid":"https://orcid.org/0000-0003-3259-9153"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Hao Zheng","raw_affiliation_strings":["National University Of Defense Technology,National Key Laboratory of Parallel and Distributed Computing,Changsha,P.R.China,410073"],"affiliations":[{"raw_affiliation_string":"National University Of Defense Technology,National Key Laboratory of Parallel and Distributed Computing,Changsha,P.R.China,410073","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061876398","display_name":"Peng Liang","orcid":"https://orcid.org/0000-0002-5590-5179"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peng Liang","raw_affiliation_strings":["National University Of Defense Technology,National Key Laboratory of Parallel and Distributed Computing,Changsha,P.R.China,410073"],"affiliations":[{"raw_affiliation_string":"National University Of Defense Technology,National Key Laboratory of Parallel and Distributed Computing,Changsha,P.R.China,410073","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102970145","display_name":"Yu Tang","orcid":"https://orcid.org/0000-0002-8595-1547"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Tang","raw_affiliation_strings":["National University Of Defense Technology,National Key Laboratory of Parallel and Distributed Computing,Changsha,P.R.China,410073"],"affiliations":[{"raw_affiliation_string":"National University Of Defense Technology,National Key Laboratory of Parallel and Distributed Computing,Changsha,P.R.China,410073","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081155669","display_name":"Yanqi Shi","orcid":"https://orcid.org/0009-0003-6845-4754"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanqi Shi","raw_affiliation_strings":["National University Of Defense Technology,National Key Laboratory of Parallel and Distributed Computing,Changsha,P.R.China,410073"],"affiliations":[{"raw_affiliation_string":"National University Of Defense Technology,National Key Laboratory of Parallel and Distributed Computing,Changsha,P.R.China,410073","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043045076","display_name":"Linbo Qiao","orcid":"https://orcid.org/0000-0002-8285-2738"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Linbo Qiao","raw_affiliation_strings":["National University Of Defense Technology,National Key Laboratory of Parallel and Distributed Computing,Changsha,P.R.China,410073"],"affiliations":[{"raw_affiliation_string":"National University Of Defense Technology,National Key Laboratory of Parallel and Distributed Computing,Changsha,P.R.China,410073","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100440903","display_name":"Dongsheng Li","orcid":"https://orcid.org/0000-0001-9743-2034"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongsheng Li","raw_affiliation_strings":["National University Of Defense Technology,National Key Laboratory of Parallel and Distributed Computing,Changsha,P.R.China,410073"],"affiliations":[{"raw_affiliation_string":"National University Of Defense Technology,National Key Laboratory of Parallel and Distributed Computing,Changsha,P.R.China,410073","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100785439"],"corresponding_institution_ids":["https://openalex.org/I170215575"],"apc_list":null,"apc_paid":null,"fwci":0.5186,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.56521184,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"6440","last_page":"6444"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9243000149726868,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9243000149726868,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11159","display_name":"Manufacturing Process and Optimization","score":0.9200999736785889,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6969738006591797},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.6845536231994629},{"id":"https://openalex.org/keywords/integer-programming","display_name":"Integer programming","score":0.6361021399497986},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.5346958041191101},{"id":"https://openalex.org/keywords/task-parallelism","display_name":"Task parallelism","score":0.43078377842903137},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.34111401438713074},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.20904657244682312}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6969738006591797},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6845536231994629},{"id":"https://openalex.org/C56086750","wikidata":"https://www.wikidata.org/wiki/Q6042592","display_name":"Integer programming","level":2,"score":0.6361021399497986},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.5346958041191101},{"id":"https://openalex.org/C42992933","wikidata":"https://www.wikidata.org/wiki/Q691169","display_name":"Task parallelism","level":3,"score":0.43078377842903137},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.34111401438713074},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.20904657244682312}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp48485.2024.10446916","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10446916","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W2908578142","https://openalex.org/W2969388332","https://openalex.org/W2991040477","https://openalex.org/W3094502228","https://openalex.org/W3129831491","https://openalex.org/W3204998121","https://openalex.org/W3205803342","https://openalex.org/W4226479682","https://openalex.org/W4310282800","https://openalex.org/W4327694855","https://openalex.org/W4372263563","https://openalex.org/W4372267501","https://openalex.org/W4375869352","https://openalex.org/W6784333009","https://openalex.org/W6811928498"],"related_works":["https://openalex.org/W2950520577","https://openalex.org/W74409296","https://openalex.org/W2003935582","https://openalex.org/W1554644772","https://openalex.org/W305742777","https://openalex.org/W2940653809","https://openalex.org/W2567390125","https://openalex.org/W2124295435","https://openalex.org/W1595672120","https://openalex.org/W4230999561"],"abstract_inverted_index":{"Transformer":[0,32],"models,":[1,33],"such":[2],"as":[3],"BERT,":[4],"GPT,":[5],"and":[6,95,119],"ViT,":[7],"have":[8,38],"been":[9,39],"applied":[10],"to":[11,21,26,52,65,124,144],"a":[12,67,76,114,121,126],"wide":[13],"range":[14],"of":[15,31,88],"areas":[16],"in":[17],"recent":[18],"years,":[19],"due":[20,51],"their":[22],"efficacy.":[23],"In":[24,71],"order":[25],"improve":[27],"the":[28,53,60,110],"training":[29,36],"efficiency":[30],"different":[34,61],"distributed":[35],"approaches":[37],"proposed,":[40],"like":[41],"Megatron-LM":[42,145],"[8].":[43,146],"However,":[44],"when":[45],"multi-dimensional":[46,128],"parallelism":[47,77,85,102],"strategies":[48,62,86],"are":[49],"considered,":[50],"complexity,":[54],"existing":[55],"works":[56],"can":[57],"not":[58],"harmonize":[59],"well":[63],"enough":[64],"obtain":[66,125],"globally":[68],"optimal":[69],"solution.":[70],"this":[72],"paper,":[73],"we":[74],"propose":[75],"strategy":[78],"searching":[79,111],"algorithm":[80],"PTIP,":[81],"which":[82],"generates":[83],"operator-level":[84],"consisting":[87],"three":[89,101],"schemes:":[90],"data":[91],"parallelism,":[92,94],"tensor":[93],"pipeline":[96],"parallelism.":[97],"PTIP":[98,136],"abstracts":[99],"these":[100],"schemes":[103],"simultaneously":[104],"into":[105,113],"an":[106],"auxiliary":[107],"graph,":[108],"reformulates":[109],"problem":[112],"mixed-integer":[115],"programming":[116],"(MIP)":[117],"problem,":[118],"uses":[120],"MIP":[122],"solver":[123],"high-quality":[127],"strategy.":[129],"Experiments":[130],"conducted":[131],"on":[132],"Transformers":[133],"demonstrate":[134],"that":[135],"obtains":[137],"13.9%":[138],"\u2212":[139],"24.7%":[140],"performance":[141],"improvement":[142],"compared":[143]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}