{"id":"https://openalex.org/W4223990029","doi":"https://doi.org/10.24963/ijcai.2022/769","title":"Survey on Efficient Training of Large Neural Networks","display_name":"Survey on Efficient Training of Large Neural Networks","publication_year":2022,"publication_date":"2022-07-01","ids":{"openalex":"https://openalex.org/W4223990029","doi":"https://doi.org/10.24963/ijcai.2022/769"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2022/769","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2022/769","pdf_url":"https://www.ijcai.org/proceedings/2022/0769.pdf","source":{"id":"https://openalex.org/S4363608755","display_name":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://www.ijcai.org/proceedings/2022/0769.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034056852","display_name":"Julia Gusak","orcid":null},"institutions":[{"id":"https://openalex.org/I125989756","display_name":"Skolkovo Institute of Science and Technology","ror":"https://ror.org/03f9nc143","country_code":"RU","type":"education","lineage":["https://openalex.org/I125989756"]}],"countries":["RU"],"is_corresponding":true,"raw_author_name":"Julia Gusak","raw_affiliation_strings":["Skoltech, Moscow, Russia"],"affiliations":[{"raw_affiliation_string":"Skoltech, Moscow, Russia","institution_ids":["https://openalex.org/I125989756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069968146","display_name":"Daria Cherniuk","orcid":null},"institutions":[{"id":"https://openalex.org/I125989756","display_name":"Skolkovo Institute of Science and Technology","ror":"https://ror.org/03f9nc143","country_code":"RU","type":"education","lineage":["https://openalex.org/I125989756"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Daria Cherniuk","raw_affiliation_strings":["Skoltech, Moscow, Russia"],"affiliations":[{"raw_affiliation_string":"Skoltech, Moscow, Russia","institution_ids":["https://openalex.org/I125989756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008247659","display_name":"Alena Shilova","orcid":"https://orcid.org/0000-0002-1795-8421"},"institutions":[{"id":"https://openalex.org/I2279609970","display_name":"Universit\u00e9 de Lille","ror":"https://ror.org/02kzqn938","country_code":"FR","type":"education","lineage":["https://openalex.org/I2279609970"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Alena Shilova","raw_affiliation_strings":["Inria, University of Lille, CRIStAL, Lille, France"],"affiliations":[{"raw_affiliation_string":"Inria, University of Lille, CRIStAL, Lille, France","institution_ids":["https://openalex.org/I2279609970"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073565062","display_name":"Alexandr Katrutsa","orcid":"https://orcid.org/0000-0002-7050-1615"},"institutions":[{"id":"https://openalex.org/I125989756","display_name":"Skolkovo Institute of Science and Technology","ror":"https://ror.org/03f9nc143","country_code":"RU","type":"education","lineage":["https://openalex.org/I125989756"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Alexandr Katrutsa","raw_affiliation_strings":["AIRI, Moscow, Russia","Skoltech, Moscow, Russia"],"affiliations":[{"raw_affiliation_string":"AIRI, Moscow, Russia","institution_ids":[]},{"raw_affiliation_string":"Skoltech, Moscow, Russia","institution_ids":["https://openalex.org/I125989756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034663545","display_name":"Daniel Bershatsky","orcid":null},"institutions":[{"id":"https://openalex.org/I125989756","display_name":"Skolkovo Institute of Science and Technology","ror":"https://ror.org/03f9nc143","country_code":"RU","type":"education","lineage":["https://openalex.org/I125989756"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Daniel Bershatsky","raw_affiliation_strings":["Skoltech, Moscow, Russia"],"affiliations":[{"raw_affiliation_string":"Skoltech, Moscow, Russia","institution_ids":["https://openalex.org/I125989756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073636091","display_name":"Xunyi Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I15057530","display_name":"Universit\u00e9 de Bordeaux","ror":"https://ror.org/057qpr032","country_code":"FR","type":"education","lineage":["https://openalex.org/I15057530"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Xunyi Zhao","raw_affiliation_strings":["Inria, University of Bordeaux, Bordeaux, France","HiePACS - High-End Parallel Algorithms for Challenging Numerical Simulations (200, avenue de la Vieille Tour \r\n33405 Talence cedex - France)"],"affiliations":[{"raw_affiliation_string":"Inria, University of Bordeaux, Bordeaux, France","institution_ids":["https://openalex.org/I15057530"]},{"raw_affiliation_string":"HiePACS - High-End Parallel Algorithms for Challenging Numerical Simulations (200, avenue de la Vieille Tour \r\n33405 Talence cedex - France)","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028817761","display_name":"Lionel Eyraud\u2010Dubois","orcid":"https://orcid.org/0000-0003-2475-3309"},"institutions":[{"id":"https://openalex.org/I15057530","display_name":"Universit\u00e9 de Bordeaux","ror":"https://ror.org/057qpr032","country_code":"FR","type":"education","lineage":["https://openalex.org/I15057530"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Lionel Eyraud-Dubois","raw_affiliation_strings":["Inria, University of Bordeaux, Bordeaux, France","HiePACS - High-End Parallel Algorithms for Challenging Numerical Simulations (200, avenue de la Vieille Tour \r\n33405 Talence cedex - France)"],"affiliations":[{"raw_affiliation_string":"Inria, University of Bordeaux, Bordeaux, France","institution_ids":["https://openalex.org/I15057530"]},{"raw_affiliation_string":"HiePACS - High-End Parallel Algorithms for Challenging Numerical Simulations (200, avenue de la Vieille Tour \r\n33405 Talence cedex - France)","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051931700","display_name":"Oleh Shliazhko","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Oleh Shliazhko","raw_affiliation_strings":["Sber, Moscow, Russia"],"affiliations":[{"raw_affiliation_string":"Sber, Moscow, Russia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009784370","display_name":"Denis Dimitrov","orcid":null},"institutions":[{"id":"https://openalex.org/I4392021246","display_name":"AIRI - Artificial Intelligence Research Institute","ror":"https://ror.org/014a87f14","country_code":null,"type":"nonprofit","lineage":["https://openalex.org/I4392021246"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Denis Dimitrov","raw_affiliation_strings":["Sber, Moscow, Russia","AIRI - Artificial Intelligence Research Institute (Russia)"],"affiliations":[{"raw_affiliation_string":"Sber, Moscow, Russia","institution_ids":[]},{"raw_affiliation_string":"AIRI - Artificial Intelligence Research Institute (Russia)","institution_ids":["https://openalex.org/I4392021246"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004111307","display_name":"Ivan Oseledets","orcid":"https://orcid.org/0000-0003-2071-2163"},"institutions":[{"id":"https://openalex.org/I125989756","display_name":"Skolkovo Institute of Science and Technology","ror":"https://ror.org/03f9nc143","country_code":"RU","type":"education","lineage":["https://openalex.org/I125989756"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Ivan Oseledets","raw_affiliation_strings":["AIRI, Moscow, Russia","Skoltech, Moscow, Russia"],"affiliations":[{"raw_affiliation_string":"AIRI, Moscow, Russia","institution_ids":[]},{"raw_affiliation_string":"Skoltech, Moscow, Russia","institution_ids":["https://openalex.org/I125989756"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064515122","display_name":"Olivier Beaumont","orcid":"https://orcid.org/0000-0003-2741-6228"},"institutions":[{"id":"https://openalex.org/I15057530","display_name":"Universit\u00e9 de Bordeaux","ror":"https://ror.org/057qpr032","country_code":"FR","type":"education","lineage":["https://openalex.org/I15057530"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Olivier Beaumont","raw_affiliation_strings":["Inria, University of Bordeaux, Bordeaux, France","HiePACS - High-End Parallel Algorithms for Challenging Numerical Simulations (200, avenue de la Vieille Tour \r\n33405 Talence cedex - France)"],"affiliations":[{"raw_affiliation_string":"Inria, University of Bordeaux, Bordeaux, France","institution_ids":["https://openalex.org/I15057530"]},{"raw_affiliation_string":"HiePACS - High-End Parallel Algorithms for Challenging Numerical Simulations (200, avenue de la Vieille Tour \r\n33405 Talence cedex - France)","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":11,"corresponding_author_ids":["https://openalex.org/A5034056852"],"corresponding_institution_ids":["https://openalex.org/I125989756"],"apc_list":null,"apc_paid":null,"fwci":0.2998,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.61671182,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"5494","last_page":"5501"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8419021368026733},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.7338722348213196},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.6422394514083862},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.6084313988685608},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5748656392097473},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.45968931913375854},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45641428232192993},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4301859140396118},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.4249327778816223},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.35928666591644287},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.32002925872802734},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.16974928975105286},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1105455756187439}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8419021368026733},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.7338722348213196},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.6422394514083862},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.6084313988685608},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5748656392097473},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.45968931913375854},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45641428232192993},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4301859140396118},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4249327778816223},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.35928666591644287},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.32002925872802734},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.16974928975105286},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1105455756187439},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.24963/ijcai.2022/769","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2022/769","pdf_url":"https://www.ijcai.org/proceedings/2022/0769.pdf","source":{"id":"https://openalex.org/S4363608755","display_name":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},{"id":"pmh:oai:HAL:hal-03952171v1","is_oa":true,"landing_page_url":"https://inria.hal.science/hal-03952171","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"https://ijcai-22.org/","raw_type":"Conference papers"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2022/769","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2022/769","pdf_url":"https://www.ijcai.org/proceedings/2022/0769.pdf","source":{"id":"https://openalex.org/S4363608755","display_name":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320324442","display_name":"Skolkovo Institute of Science and Technology","ror":"https://ror.org/03f9nc143"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4223990029.pdf","grobid_xml":"https://content.openalex.org/works/W4223990029.grobid-xml"},"referenced_works_count":46,"referenced_works":["https://openalex.org/W90882117","https://openalex.org/W1585773866","https://openalex.org/W2338908902","https://openalex.org/W2489529491","https://openalex.org/W2622263826","https://openalex.org/W2757910899","https://openalex.org/W2769644379","https://openalex.org/W2809901035","https://openalex.org/W2946900926","https://openalex.org/W2947664806","https://openalex.org/W2963766684","https://openalex.org/W2963959597","https://openalex.org/W2970587001","https://openalex.org/W2981237538","https://openalex.org/W2991040477","https://openalex.org/W2991615366","https://openalex.org/W2995435108","https://openalex.org/W3006131567","https://openalex.org/W3035655392","https://openalex.org/W3036879053","https://openalex.org/W3037639655","https://openalex.org/W3038581078","https://openalex.org/W3092902420","https://openalex.org/W3100141945","https://openalex.org/W3101100041","https://openalex.org/W3103522166","https://openalex.org/W3127394918","https://openalex.org/W3129831491","https://openalex.org/W3132977829","https://openalex.org/W3137550677","https://openalex.org/W3174394143","https://openalex.org/W3180473732","https://openalex.org/W3203426023","https://openalex.org/W3204105967","https://openalex.org/W3204921648","https://openalex.org/W3204998121","https://openalex.org/W3206606249","https://openalex.org/W4221148934","https://openalex.org/W4221154625","https://openalex.org/W4286909525","https://openalex.org/W4287214056","https://openalex.org/W4287756266","https://openalex.org/W4287774547","https://openalex.org/W4288017660","https://openalex.org/W4293404878","https://openalex.org/W4306878280"],"related_works":["https://openalex.org/W2120447654","https://openalex.org/W2977179488","https://openalex.org/W2144453115","https://openalex.org/W2128223750","https://openalex.org/W4238532390","https://openalex.org/W2188872161","https://openalex.org/W2961779879","https://openalex.org/W797688974","https://openalex.org/W3000197790","https://openalex.org/W4285818394"],"abstract_inverted_index":{"Modern":[0],"Deep":[1],"Neural":[2],"Networks":[3],"(DNNs)":[4],"require":[5],"significant":[6],"memory":[7,57],"to":[8],"store":[9],"weight,":[10],"activations,":[11],"and":[12,58,64,82,86],"other":[13],"intermediate":[14],"tensors":[15],"during":[16],"training.":[17,51],"Hence,":[18],"many":[19],"models":[20],"don\u2019t":[21],"fit":[22],"one":[23],"GPU":[24],"device":[25],"or":[26,72],"can":[27],"be":[28],"trained":[29],"using":[30],"only":[31],"a":[32,40,70],"small":[33],"per-GPU":[34],"batch":[35],"size.":[36],"This":[37],"survey":[38],"provides":[39],"systematic":[41],"overview":[42],"of":[43,62,80],"the":[44,77,94],"approaches":[45,91],"that":[46,55],"enable":[47],"more":[48],"efficient":[49],"DNNs":[50],"We":[52,75],"analyze":[53],"techniques":[54],"save":[56],"make":[59],"good":[60],"use":[61],"computation":[63],"communication":[65],"resources":[66],"on":[67],"architectures":[68],"with":[69,90],"single":[71],"several":[73],"GPUs.":[74],"summarize":[76],"main":[78],"categories":[79],"strategies":[81,84],"compare":[83],"within":[85],"across":[87],"categories.":[88],"Along":[89],"proposed":[92],"in":[93],"literature,":[95],"we":[96],"discuss":[97],"available":[98],"implementations.":[99]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2022-04-19T00:00:00"}