{"id":"https://openalex.org/W2912759934","doi":"https://doi.org/10.1109/tpds.2018.2866582","title":"moDNN: Memory Optimal Deep Neural Network Training on Graphics Processing Units","display_name":"moDNN: Memory Optimal Deep Neural Network Training on Graphics Processing Units","publication_year":2018,"publication_date":"2018-08-23","ids":{"openalex":"https://openalex.org/W2912759934","doi":"https://doi.org/10.1109/tpds.2018.2866582","mag":"2912759934"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2018.2866582","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2018.2866582","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Xiaoming Chen","orcid":"https://orcid.org/0000-0001-6565-2884"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoming Chen","raw_affiliation_strings":["State Key Laboratory of Computer Architecture, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6565-2884","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Computer Architecture, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Danny Ziyi Chen","orcid":"https://orcid.org/0000-0002-7337-1844"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Danny Ziyi Chen","raw_affiliation_strings":["Department of Computer Science and Engineering, University of Notre Dame, Notre Dame, IN, USA"],"raw_orcid":"https://orcid.org/0000-0002-7337-1844","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University of Notre Dame, Notre Dame, IN, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016864694","display_name":"Yinhe Han","orcid":"https://orcid.org/0000-0003-0904-6681"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yinhe Han","raw_affiliation_strings":["State Key Laboratory of Computer Architecture, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0904-6681","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Computer Architecture, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100600905","display_name":"Xiaobo Sharon Hu","orcid":"https://orcid.org/0000-0002-6636-9738"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaobo Sharon Hu","raw_affiliation_strings":["Department of Computer Science and Engineering, University of Notre Dame, Notre Dame, IN, USA"],"raw_orcid":"https://orcid.org/0000-0002-6636-9738","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University of Notre Dame, Notre Dame, IN, USA","institution_ids":["https://openalex.org/I107639228"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2645,"currency":"USD","value_usd":2645},"apc_paid":null,"fwci":0.9821,"has_fulltext":false,"cited_by_count":19,"citation_normalized_percentile":{"value":0.78487731,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"30","issue":"3","first_page":"646","last_page":"661"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.9118683338165283},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.6934174299240112},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.6653273105621338},{"id":"https://openalex.org/keywords/graphics","display_name":"Graphics","score":0.5148996114730835},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.5122050642967224},{"id":"https://openalex.org/keywords/high-memory","display_name":"High memory","score":0.4776136577129364},{"id":"https://openalex.org/keywords/memory-management","display_name":"Memory management","score":0.4772638976573944},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.4710053503513336},{"id":"https://openalex.org/keywords/graphics-processing-unit","display_name":"Graphics processing unit","score":0.4350661635398865},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.43194451928138733},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.357022225856781},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.20169958472251892},{"id":"https://openalex.org/keywords/computer-hardware","display_name":"Computer hardware","score":0.18665647506713867},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.11767366528511047},{"id":"https://openalex.org/keywords/semiconductor-memory","display_name":"Semiconductor memory","score":0.09992033243179321}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.9118683338165283},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6934174299240112},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.6653273105621338},{"id":"https://openalex.org/C21442007","wikidata":"https://www.wikidata.org/wiki/Q1027879","display_name":"Graphics","level":2,"score":0.5148996114730835},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.5122050642967224},{"id":"https://openalex.org/C2781357197","wikidata":"https://www.wikidata.org/wiki/Q5757597","display_name":"High memory","level":2,"score":0.4776136577129364},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.4772638976573944},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.4710053503513336},{"id":"https://openalex.org/C2779851693","wikidata":"https://www.wikidata.org/wiki/Q183484","display_name":"Graphics processing unit","level":2,"score":0.4350661635398865},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.43194451928138733},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.357022225856781},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.20169958472251892},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.18665647506713867},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.11767366528511047},{"id":"https://openalex.org/C98986596","wikidata":"https://www.wikidata.org/wiki/Q1143031","display_name":"Semiconductor memory","level":2,"score":0.09992033243179321}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2018.2866582","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2018.2866582","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2818552058","display_name":null,"funder_award_id":"CCF-1617735","funder_id":"https://openalex.org/F4320335353","funder_display_name":"National Science Foundation of Sri Lanka"},{"id":"https://openalex.org/G3772543219","display_name":null,"funder_award_id":"CCF-1217906","funder_id":"https://openalex.org/F4320335353","funder_display_name":"National Science Foundation of Sri Lanka"},{"id":"https://openalex.org/G5129848219","display_name":null,"funder_award_id":"CNS-1629914","funder_id":"https://openalex.org/F4320335353","funder_display_name":"National Science Foundation of Sri Lanka"},{"id":"https://openalex.org/G7586553789","display_name":null,"funder_award_id":"CCF-1640081","funder_id":"https://openalex.org/F4320335353","funder_display_name":"National Science Foundation of Sri Lanka"}],"funders":[{"id":"https://openalex.org/F4320306087","display_name":"Semiconductor Research Corporation","ror":"https://ror.org/047z4n946"},{"id":"https://openalex.org/F4320335353","display_name":"National Science Foundation of Sri Lanka","ror":"https://ror.org/010xaa060"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":76,"referenced_works":["https://openalex.org/W128208790","https://openalex.org/W753012316","https://openalex.org/W1590012787","https://openalex.org/W1667652561","https://openalex.org/W1686810756","https://openalex.org/W1841592590","https://openalex.org/W1901129140","https://openalex.org/W1903029394","https://openalex.org/W1935978687","https://openalex.org/W1972501001","https://openalex.org/W2016879831","https://openalex.org/W2046441184","https://openalex.org/W2097117768","https://openalex.org/W2102017903","https://openalex.org/W2102605133","https://openalex.org/W2152175008","https://openalex.org/W2155893237","https://openalex.org/W2157282184","https://openalex.org/W2160815625","https://openalex.org/W2163605009","https://openalex.org/W2168231600","https://openalex.org/W2172654076","https://openalex.org/W2186615578","https://openalex.org/W2194775991","https://openalex.org/W2271840356","https://openalex.org/W2286365479","https://openalex.org/W2300242332","https://openalex.org/W2317002898","https://openalex.org/W2319920447","https://openalex.org/W2332506150","https://openalex.org/W2338908902","https://openalex.org/W2489529491","https://openalex.org/W2562749854","https://openalex.org/W2617766261","https://openalex.org/W2798707674","https://openalex.org/W2884834742","https://openalex.org/W2962776038","https://openalex.org/W2963042536","https://openalex.org/W2963340555","https://openalex.org/W2963374099","https://openalex.org/W2963456262","https://openalex.org/W2963674932","https://openalex.org/W2964174152","https://openalex.org/W3145128584","https://openalex.org/W3152703111","https://openalex.org/W4246219036","https://openalex.org/W4293404878","https://openalex.org/W4297785678","https://openalex.org/W4297827580","https://openalex.org/W4302296459","https://openalex.org/W6622239757","https://openalex.org/W6637151318","https://openalex.org/W6637373629","https://openalex.org/W6638632666","https://openalex.org/W6638783484","https://openalex.org/W6639824700","https://openalex.org/W6640289440","https://openalex.org/W6640442106","https://openalex.org/W6675112735","https://openalex.org/W6682825348","https://openalex.org/W6684191040","https://openalex.org/W6684859321","https://openalex.org/W6686509673","https://openalex.org/W6694517276","https://openalex.org/W6696004547","https://openalex.org/W6698200048","https://openalex.org/W6700264148","https://openalex.org/W6702429100","https://openalex.org/W6703652217","https://openalex.org/W6717768942","https://openalex.org/W6732127223","https://openalex.org/W6738460352","https://openalex.org/W6750715941","https://openalex.org/W6753668332","https://openalex.org/W6763697522","https://openalex.org/W7057659579"],"related_works":["https://openalex.org/W2058965144","https://openalex.org/W2348330046","https://openalex.org/W2141978748","https://openalex.org/W2612219836","https://openalex.org/W4200309432","https://openalex.org/W1981291981","https://openalex.org/W4253567283","https://openalex.org/W2567721291","https://openalex.org/W2896122000","https://openalex.org/W1527643973"],"abstract_inverted_index":{"Graphics":[0],"processing":[1],"units":[2],"(GPUs)":[3],"have":[4],"been":[5,24],"widely":[6],"adopted":[7],"to":[8,60,69,85,112,125,150,165],"accelerate":[9],"the":[10,18,27,39,71,94,160,179,196,215],"training":[11,66,83,180],"of":[12,21,30,41,81,102,217],"deep":[13],"neural":[14],"networks":[15],"(DNNs).":[16],"Although":[17],"computational":[19],"performance":[20],"GPUs":[22,32,211],"has":[23],"improving":[25],"steadily,":[26],"memory":[28,72,89,127,142,147,162],"size":[29,135],"modern":[31],"is":[33,163,182,189,230],"still":[34],"quite":[35],"limited,":[36],"which":[37,138,157,188],"restricts":[38],"sizes":[40],"DNNs":[42],"that":[43,159,193,227],"can":[44,145],"be":[45],"trained":[46],"on":[47,67,172,209,232],"GPUs,":[48],"and":[49,105,117,221],"hence":[50],"raises":[51],"serious":[52],"challenges.":[53],"This":[54],"paper":[55],"introduces":[56],"a":[57,132,173],"framework,":[58],"referred":[59],"as":[61],"moDNN":[62,77,144,171,208],"(memory":[63],"optimal":[64],"DNN":[65,75,82],"GPUs),":[68],"optimize":[70,126],"usage":[73,148],"in":[74],"training.":[76],"supports":[78],"automatic":[79],"tuning":[80],"code":[84],"match":[86],"any":[87],"given":[88],"budget":[90],"(not":[91],"smaller":[92],"than":[93,192],"theoretical":[95],"lower":[96],"bound).":[97],"By":[98],"taking":[99],"full":[100],"advantage":[101],"overlapping":[103,218],"computations":[104],"data":[106,115,219],"transfers,":[107,119],"we":[108,202],"develop":[109],"new":[110,133],"heuristics":[111],"judiciously":[113],"schedule":[114],"offloading":[116],"prefetching":[118],"together":[120],"with":[121,153,175],"convolution":[122],"algorithm":[123],"selection,":[124],"usage.":[128,143],"We":[129],"further":[130],"devise":[131],"sub-batch":[134],"selection":[136],"method":[137],"also":[139],"greatly":[140],"reduces":[141],"save":[146],"up":[149],"59\u00d7,":[151],"compared":[152],"an":[154,204],"ideal":[155],"case":[156],"assumes":[158],"GPU":[161,174,222],"sufficient":[164],"hold":[166],"all":[167],"data.":[168],"When":[169],"executing":[170],"12":[176],"GB":[177],"memory,":[178],"time":[181],"increased":[183],"by":[184,195,213],"only":[185],"3":[186],"percent,":[187],"much":[190],"shorter":[191],"incurred":[194],"best":[197],"known":[198],"approach,":[199],"vDNN.":[200],"Furthermore,":[201],"propose":[203],"optimization":[205],"strategy":[206],"for":[207],"multiple":[210],"again":[212],"utilizing":[214],"idea":[216],"transfers":[220],"computations.":[223],"The":[224],"results":[225],"show":[226],"3.7\u00d7":[228],"speedup":[229],"attained":[231],"four":[233],"GPUs.":[234]},"counts_by_year":[{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":2}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}