{"id":"https://openalex.org/W3202134982","doi":"https://doi.org/10.1145/3472456.3472497","title":"Hippie: A Data-Paralleled Pipeline Approach to Improve Memory-Efficiency and Scalability for Large DNN Training","display_name":"Hippie: A Data-Paralleled Pipeline Approach to Improve Memory-Efficiency and Scalability for Large DNN Training","publication_year":2021,"publication_date":"2021-08-09","ids":{"openalex":"https://openalex.org/W3202134982","doi":"https://doi.org/10.1145/3472456.3472497","mag":"3202134982"},"language":"en","primary_location":{"id":"doi:10.1145/3472456.3472497","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3472456.3472497","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"50th International Conference on Parallel Processing","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114546959","display_name":"Xiangyu Ye","orcid":"https://orcid.org/0009-0008-1285-5176"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xiangyu Ye","raw_affiliation_strings":["National University of Defense Technology, China"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055426388","display_name":"Zhiquan Lai","orcid":"https://orcid.org/0000-0002-3458-4732"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiquan Lai","raw_affiliation_strings":["National University of Defense Technology, China"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106623069","display_name":"Shengwei Li","orcid":"https://orcid.org/0000-0002-7419-1511"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shengwei Li","raw_affiliation_strings":["National University of Defense Technology, China"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114546953","display_name":"Lei Cai","orcid":"https://orcid.org/0009-0007-6965-2268"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Cai","raw_affiliation_strings":["National University of Defense Technology, China"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058308008","display_name":"Ding Sun","orcid":"https://orcid.org/0000-0002-8494-1574"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ding Sun","raw_affiliation_strings":["National University of Defense Technology, China"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043045076","display_name":"Linbo Qiao","orcid":"https://orcid.org/0000-0002-8285-2738"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Linbo Qiao","raw_affiliation_strings":["National University of Defense Technology, China"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100440903","display_name":"Dongsheng Li","orcid":"https://orcid.org/0000-0001-9743-2034"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongsheng Li","raw_affiliation_strings":["National University of Defense Technology, China"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology, China","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5114546959"],"corresponding_institution_ids":["https://openalex.org/I170215575"],"apc_list":null,"apc_paid":null,"fwci":1.3565,"has_fulltext":false,"cited_by_count":20,"citation_normalized_percentile":{"value":0.83274448,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8552742600440979},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7825430631637573},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.7046611309051514},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6702892184257507},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.6397837400436401},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.5718768239021301},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.5062590837478638},{"id":"https://openalex.org/keywords/data-parallelism","display_name":"Data parallelism","score":0.5029568076133728},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.3689703941345215},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.35173043608665466},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.08324882388114929}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8552742600440979},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7825430631637573},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.7046611309051514},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6702892184257507},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.6397837400436401},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.5718768239021301},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.5062590837478638},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.5029568076133728},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.3689703941345215},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.35173043608665466},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.08324882388114929},{"id":"https://openalex.org/C555944384","wikidata":"https://www.wikidata.org/wiki/Q249","display_name":"Wireless","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3472456.3472497","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3472456.3472497","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"50th International Conference on Parallel Processing","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Affordable and clean energy","id":"https://metadata.un.org/sdg/7","score":0.5400000214576721}],"awards":[{"id":"https://openalex.org/G3668830066","display_name":null,"funder_award_id":"No.2018YFB0204300","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W1528469369","https://openalex.org/W1598866093","https://openalex.org/W1686810756","https://openalex.org/W1833917188","https://openalex.org/W1967134278","https://openalex.org/W1975934607","https://openalex.org/W2113547287","https://openalex.org/W2117539524","https://openalex.org/W2271840356","https://openalex.org/W2335835108","https://openalex.org/W2512971201","https://openalex.org/W2734941459","https://openalex.org/W2766166018","https://openalex.org/W2884700152","https://openalex.org/W2886288284","https://openalex.org/W2906007643","https://openalex.org/W2944793600","https://openalex.org/W2951781666","https://openalex.org/W2962801832","https://openalex.org/W2963601856","https://openalex.org/W2965658867","https://openalex.org/W2969388332","https://openalex.org/W2969766737","https://openalex.org/W2991615366","https://openalex.org/W3036879053","https://openalex.org/W3089631566","https://openalex.org/W3103894541","https://openalex.org/W3132107458","https://openalex.org/W4301361180"],"related_works":["https://openalex.org/W2375218795","https://openalex.org/W2078379271","https://openalex.org/W2393010557","https://openalex.org/W2043460294","https://openalex.org/W2950520577","https://openalex.org/W1554644772","https://openalex.org/W2003935582","https://openalex.org/W2494130044","https://openalex.org/W2033862586","https://openalex.org/W74409296"],"abstract_inverted_index":{"With":[0],"the":[1,39,51,86,108,117,129,134,151,164],"increase":[2],"of":[3,42,53,91,112,136,150,158,206],"both":[4],"data":[5,28,82],"and":[6,32,63,81,89,110,121,132,168,177],"parameter":[7],"volume,":[8],"it":[9],"has":[10],"become":[11],"a":[12,72,97,146,192],"big":[13],"challenge":[14],"to":[15,84,126,162,201],"efficiently":[16],"train":[17],"large-scale":[18],"DNN":[19,44,93,139],"models":[20,140,182],"on":[21,47,102,141,175,180,191],"distributed":[22],"platforms.":[23],"Ordinary":[24],"parallelism":[25,31,80,83],"modes,":[26],"i.e.,":[27],"parallelism,":[29,34],"model":[30,45],"pipeline":[33,79,119],"can":[35],"no":[36],"longer":[37],"satisfy":[38],"efficient":[40],"scaling":[41,189],"large":[43,92,138],"training":[46,64,75,137],"multiple":[48],"nodes.":[49],"Meanwhile,":[50,114],"problem":[52],"too":[54],"much":[55],"memory":[56,87,130,159,169,207,212],"consumption":[57],"seriously":[58],"restricts":[59],"GPU":[60],"computing":[61],"efficiency":[62,88,160,190],"throughput.":[65],"In":[66],"this":[67],"paper,":[68],"we":[69,154],"propose":[70,155],"Hippie,":[71],"hybrid":[73,98],"parallel":[74,99],"framework":[76],"that":[77,184],"integrates":[78],"improve":[85],"scalability":[90,111],"training.":[94,113],"Hippie":[95,115,173,185,196],"adopts":[96],"method":[100],"based":[101,174],"hiding":[103],"gradient":[104],"communication,":[105],"which":[106],"improves":[107],"throughput":[109,167,198],"introduces":[116],"last-stage":[118],"scheduling":[120],"recomputation":[122],"for":[123],"specific":[124],"layers":[125],"effectively":[127],"reduce":[128],"overhead":[131],"ease":[133],"difficulties":[135],"memory-constrained":[142],"devices.":[143],"To":[144],"achieve":[145],"more":[147],"reasonable":[148],"evaluation":[149],"optimization":[152],"effect,":[153],"an":[156],"index":[157],"(ME)":[161],"represent":[163],"tradeoff":[165],"between":[166],"overhead.":[170],"We":[171],"implement":[172],"PyTorch":[176],"NCCL.":[178],"Experiments":[179],"various":[181],"show":[183],"achieves":[186],"above":[187],"90%":[188],"16-GPU":[193],"platform.":[194],"Moreover,":[195],"increases":[197],"by":[199],"up":[200],"80%":[202],"while":[203],"saving":[204],"57%":[205],"overhead,":[208],"achieving":[209],"4.18":[210],"\u00d7":[211],"efficiency.":[213]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":2}],"updated_date":"2026-03-25T14:56:36.534964","created_date":"2025-10-10T00:00:00"}