{"id":"https://openalex.org/W4384787320","doi":"https://doi.org/10.1109/access.2023.3296609","title":"Accelerated Synchronous Model Parallelism Using Cooperative Process for Training Compute-Intensive Models","display_name":"Accelerated Synchronous Model Parallelism Using Cooperative Process for Training Compute-Intensive Models","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4384787320","doi":"https://doi.org/10.1109/access.2023.3296609"},"language":"en","primary_location":{"id":"doi:10.1109/access.2023.3296609","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3296609","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10185201.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10185201.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032308190","display_name":"Chanhee Yu","orcid":"https://orcid.org/0000-0003-0811-5542"},"institutions":[{"id":"https://openalex.org/I878022262","display_name":"Korea Institute of Science & Technology Information","ror":"https://ror.org/01k4yrm29","country_code":"KR","type":"facility","lineage":["https://openalex.org/I2801339556","https://openalex.org/I4210144908","https://openalex.org/I4387152098","https://openalex.org/I878022262"]},{"id":"https://openalex.org/I88761825","display_name":"Korea University of Science and Technology","ror":"https://ror.org/000qzf213","country_code":"KR","type":"education","lineage":["https://openalex.org/I88761825"]}],"countries":["KR"],"is_corresponding":true,"raw_author_name":"Chanhee Yu","raw_affiliation_strings":["Department of Big Data Science, University of Science and Technology, Daejeon, South Korea","Center for Supercomputing Technology Development, Korea Institute of Science and Technology Information, Daejeon, Korea"],"affiliations":[{"raw_affiliation_string":"Department of Big Data Science, University of Science and Technology, Daejeon, South Korea","institution_ids":["https://openalex.org/I88761825"]},{"raw_affiliation_string":"Center for Supercomputing Technology Development, Korea Institute of Science and Technology Information, Daejeon, Korea","institution_ids":["https://openalex.org/I878022262"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5003111778","display_name":"Kyongseok Park","orcid":"https://orcid.org/0000-0001-5164-7099"},"institutions":[{"id":"https://openalex.org/I878022262","display_name":"Korea Institute of Science & Technology Information","ror":"https://ror.org/01k4yrm29","country_code":"KR","type":"facility","lineage":["https://openalex.org/I2801339556","https://openalex.org/I4210144908","https://openalex.org/I4387152098","https://openalex.org/I878022262"]},{"id":"https://openalex.org/I88761825","display_name":"Korea University of Science and Technology","ror":"https://ror.org/000qzf213","country_code":"KR","type":"education","lineage":["https://openalex.org/I88761825"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Kyongseok Park","raw_affiliation_strings":["Department of Applied AI, University of Science and Technology, Daejeon, South Korea","Center for Supercomputing Technology Development, Korea Institute of Science and Technology Information, Daejeon, Korea"],"affiliations":[{"raw_affiliation_string":"Department of Applied AI, University of Science and Technology, Daejeon, South Korea","institution_ids":["https://openalex.org/I88761825"]},{"raw_affiliation_string":"Center for Supercomputing Technology Development, Korea Institute of Science and Technology Information, Daejeon, Korea","institution_ids":["https://openalex.org/I878022262"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5032308190"],"corresponding_institution_ids":["https://openalex.org/I878022262","https://openalex.org/I88761825"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.4776,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.64320578,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"11","issue":null,"first_page":"74914","last_page":"74923"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9894999861717224,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8614124655723572},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.7077959775924683},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.6536266803741455},{"id":"https://openalex.org/keywords/acceleration","display_name":"Acceleration","score":0.6197760701179504},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.6179468035697937},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.5622318983078003},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5313594341278076},{"id":"https://openalex.org/keywords/idle","display_name":"Idle","score":0.4646197557449341},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.24218201637268066},{"id":"https://openalex.org/keywords/channel","display_name":"Channel (broadcasting)","score":0.14284777641296387}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8614124655723572},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.7077959775924683},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.6536266803741455},{"id":"https://openalex.org/C117896860","wikidata":"https://www.wikidata.org/wiki/Q11376","display_name":"Acceleration","level":2,"score":0.6197760701179504},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.6179468035697937},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.5622318983078003},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5313594341278076},{"id":"https://openalex.org/C16320812","wikidata":"https://www.wikidata.org/wiki/Q1812200","display_name":"Idle","level":2,"score":0.4646197557449341},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.24218201637268066},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.14284777641296387},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C74650414","wikidata":"https://www.wikidata.org/wiki/Q11397","display_name":"Classical mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2023.3296609","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3296609","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10185201.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:270bccce13bf448ea919105531da6cf8","is_oa":true,"landing_page_url":"https://doaj.org/article/270bccce13bf448ea919105531da6cf8","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 11, Pp 74914-74923 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2023.3296609","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3296609","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10185201.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1092443052","display_name":null,"funder_award_id":"S3126610","funder_id":"https://openalex.org/F4320327819","funder_display_name":"Ministry of SMEs and Startups"},{"id":"https://openalex.org/G2096008949","display_name":null,"funder_award_id":"K-23-L02-C07-S01","funder_id":"https://openalex.org/F4320322105","funder_display_name":"Korea Institute of Science and Technology Information"},{"id":"https://openalex.org/G4517773220","display_name":null,"funder_award_id":"S3126610","funder_id":"https://openalex.org/F4320335977","funder_display_name":"Korea Technology and Information Promotion Agency for SMEs"},{"id":"https://openalex.org/G722745221","display_name":null,"funder_award_id":"S3126610","funder_id":"https://openalex.org/F4320322105","funder_display_name":"Korea Institute of Science and Technology Information"}],"funders":[{"id":"https://openalex.org/F4320322030","display_name":"Ministry of Science, ICT and Future Planning","ror":"https://ror.org/032e49973"},{"id":"https://openalex.org/F4320322105","display_name":"Korea Institute of Science and Technology Information","ror":"https://ror.org/01k4yrm29"},{"id":"https://openalex.org/F4320327819","display_name":"Ministry of SMEs and Startups","ror":null},{"id":"https://openalex.org/F4320335977","display_name":"Korea Technology and Information Promotion Agency for SMEs","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4384787320.pdf","grobid_xml":"https://content.openalex.org/works/W4384787320.grobid-xml"},"referenced_works_count":45,"referenced_works":["https://openalex.org/W1686810756","https://openalex.org/W2131613942","https://openalex.org/W2163605009","https://openalex.org/W2168231600","https://openalex.org/W2183341477","https://openalex.org/W2186615578","https://openalex.org/W2194775991","https://openalex.org/W2592962403","https://openalex.org/W2896457183","https://openalex.org/W2954996726","https://openalex.org/W2969388332","https://openalex.org/W2972087877","https://openalex.org/W2982664135","https://openalex.org/W2985738161","https://openalex.org/W2991040477","https://openalex.org/W3036879053","https://openalex.org/W3092310353","https://openalex.org/W3109982791","https://openalex.org/W3122890974","https://openalex.org/W3130395060","https://openalex.org/W3132107458","https://openalex.org/W3137550677","https://openalex.org/W3143293307","https://openalex.org/W3164436820","https://openalex.org/W3188270315","https://openalex.org/W3204998121","https://openalex.org/W3205803342","https://openalex.org/W3206832494","https://openalex.org/W4224035244","https://openalex.org/W4288365403","https://openalex.org/W4301239768","https://openalex.org/W4312910503","https://openalex.org/W6637373629","https://openalex.org/W6684191040","https://openalex.org/W6684859321","https://openalex.org/W6686509673","https://openalex.org/W6748645090","https://openalex.org/W6755207826","https://openalex.org/W6756718674","https://openalex.org/W6762407294","https://openalex.org/W6769275798","https://openalex.org/W6769865700","https://openalex.org/W6779068807","https://openalex.org/W6779863968","https://openalex.org/W6790665546"],"related_works":["https://openalex.org/W2317245370","https://openalex.org/W4249323025","https://openalex.org/W198851386","https://openalex.org/W2030310580","https://openalex.org/W947442053","https://openalex.org/W1980160788","https://openalex.org/W2148915962","https://openalex.org/W2283866686","https://openalex.org/W4287182096","https://openalex.org/W27867058"],"abstract_inverted_index":{"As":[0],"deep":[1],"learning":[2],"has":[3,50],"been":[4],"recently":[5],"applied":[6],"to":[7,34,57,73,92,128,201,209,244],"a":[8,15,36,51,69,75,93],"wide":[9],"variety":[10],"of":[11,44,96,139,186,218],"fields,":[12],"there":[13],"is":[14,77,89],"growing":[16],"demand":[17],"for":[18,142,238],"models":[19,229],"that":[20],"can":[21,174],"handle":[22],"large":[23],"input":[24,137],"data,":[25],"including":[26],"high-resolution":[27],"images.":[28],"Therefore,":[29],"model":[30,37],"parallelism":[31],"was":[32,240],"proposed":[33,117,191,213],"train":[35,129],"whose":[38],"size":[39],"exceeds":[40],"the":[41,65,83,86,100,102,104,106,114,133,136,143,150,154,160,183,187,190,203,210,212,224,235],"memory":[42,195,236],"capacity":[43],"an":[45,169],"accelerator,":[46],"but":[47],"this":[48,61],"method":[49,192,214],"very":[52],"slow":[53],"training":[54,87,239],"speed":[55,88],"due":[56],"bubbles.":[58],"To":[59,112],"solve":[60],"problem,":[62],"GPipe":[63],"reduced":[64,241],"bubbles":[66,180],"by":[67,181,242],"proposing":[68],"micro-batch":[70,97],"concept,":[71],"according":[72],"which":[74],"mini-batch":[76],"divided":[78],"into":[79,164],"smaller":[80,101],"units.":[81],"However,":[82],"improvement":[84],"in":[85,153,157,205],"still":[90],"limited":[91],"certain":[94],"extent":[95],"size,":[98],"because":[99,196],"micro-batches,":[103],"less":[105],"computation":[107],"and":[108,121,159,178,221,227,234],"input/output":[109],"(I/O)":[110],"efficiency.":[111],"overcome":[113],"limitations,":[115],"we":[116],"acceleration":[118],"through":[119],"prediction":[120,134],"synchronization":[122,155],"steps":[123],"based":[124],"on":[125],"process":[126,167],"cooperation":[127],"compute-intensive":[130],"models.":[131],"In":[132],"step,":[135],"data":[138],"all":[140],"processes":[141],"forward":[144],"pass":[145],"are":[146,162],"calculated":[147],"concurrently":[148],"using":[149],"weights":[151],"shared":[152],"step":[156],"advance,":[158],"results":[161],"gathered":[163],"each":[165],"corresponding":[166],"via":[168],"all-to-all":[170],"collective":[171],"operation.":[172],"This":[173],"increase":[175],"computational":[176],"efficiency":[177],"reduce":[179],"minimizing":[182],"idle":[184],"state":[185],"device.":[188],"Additionally,":[189],"requires":[193],"minimal":[194],"it":[197],"does":[198],"not":[199],"have":[200],"store":[202],"activations":[204],"memory.":[206],"Thus,":[207],"compared":[208],"GPipe,":[211],"achieved":[215],"performance":[216],"improvements":[217],"15.3%,":[219],"34.5%,":[220],"25.8%":[222],"with":[223,230],"VGG16bn,":[225],"ResNet50,":[226],"InceptionV3":[228],"four":[231],"devices,":[232],"respectively,":[233],"used":[237],"up":[243],"75.0%.":[245]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}