{"id":"https://openalex.org/W3205803342","doi":"https://doi.org/10.1145/3458817.3476205","title":"ZeRO-infinity","display_name":"ZeRO-infinity","publication_year":2021,"publication_date":"2021-10-21","ids":{"openalex":"https://openalex.org/W3205803342","doi":"https://doi.org/10.1145/3458817.3476205","mag":"3205803342"},"language":"en","primary_location":{"id":"doi:10.1145/3458817.3476205","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3458817.3476205","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069595628","display_name":"Samyam Rajbhandari","orcid":"https://orcid.org/0000-0002-0386-8759"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Samyam Rajbhandari","raw_affiliation_strings":[""],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022644245","display_name":"Olatunji Ruwase","orcid":"https://orcid.org/0000-0002-5508-0728"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Olatunji Ruwase","raw_affiliation_strings":[""],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037534081","display_name":"Jeff Rasley","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jeff Rasley","raw_affiliation_strings":[""],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089950471","display_name":"Shaden Smith","orcid":"https://orcid.org/0000-0003-4072-9990"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shaden Smith","raw_affiliation_strings":[""],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040302174","display_name":"Yuxiong He","orcid":"https://orcid.org/0000-0003-0478-8854"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuxiong He","raw_affiliation_strings":[""],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":30.2137,"has_fulltext":false,"cited_by_count":219,"citation_normalized_percentile":{"value":0.99923454,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"14"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7389620542526245},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.6968569755554199},{"id":"https://openalex.org/keywords/zero","display_name":"Zero (linguistics)","score":0.48316067457199097},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.4788215756416321},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.46346110105514526},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.4593830704689026},{"id":"https://openalex.org/keywords/computational-science","display_name":"Computational science","score":0.45249128341674805},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.08860734105110168},{"id":"https://openalex.org/keywords/nanotechnology","display_name":"Nanotechnology","score":0.06991651654243469}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7389620542526245},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6968569755554199},{"id":"https://openalex.org/C2780813799","wikidata":"https://www.wikidata.org/wiki/Q3274237","display_name":"Zero (linguistics)","level":2,"score":0.48316067457199097},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.4788215756416321},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.46346110105514526},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.4593830704689026},{"id":"https://openalex.org/C459310","wikidata":"https://www.wikidata.org/wiki/Q117801","display_name":"Computational science","level":1,"score":0.45249128341674805},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.08860734105110168},{"id":"https://openalex.org/C171250308","wikidata":"https://www.wikidata.org/wiki/Q11468","display_name":"Nanotechnology","level":1,"score":0.06991651654243469},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3458817.3476205","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3458817.3476205","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.46000000834465027,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":39,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W2146502635","https://openalex.org/W2489529491","https://openalex.org/W2612387305","https://openalex.org/W2763421725","https://openalex.org/W2783292547","https://openalex.org/W2788193959","https://openalex.org/W2807147113","https://openalex.org/W2883283076","https://openalex.org/W2883830791","https://openalex.org/W2891993230","https://openalex.org/W2896457183","https://openalex.org/W2901299405","https://openalex.org/W2973727699","https://openalex.org/W2979044977","https://openalex.org/W2981852735","https://openalex.org/W3001279689","https://openalex.org/W3010830594","https://openalex.org/W3010969086","https://openalex.org/W3012479151","https://openalex.org/W3012514909","https://openalex.org/W3101104221","https://openalex.org/W3103894541","https://openalex.org/W3121562065","https://openalex.org/W3129831491","https://openalex.org/W3157864729","https://openalex.org/W4288089799","https://openalex.org/W4301361180","https://openalex.org/W6600297362","https://openalex.org/W6703652217","https://openalex.org/W6739901393","https://openalex.org/W6743289643","https://openalex.org/W6745245109","https://openalex.org/W6756379755","https://openalex.org/W6760810938","https://openalex.org/W6761628794","https://openalex.org/W6772383348","https://openalex.org/W6776048684","https://openalex.org/W6862640317"],"related_works":["https://openalex.org/W2045183646","https://openalex.org/W2162409446","https://openalex.org/W2109463584","https://openalex.org/W2504075107","https://openalex.org/W2354058185","https://openalex.org/W3014500535","https://openalex.org/W2319812372","https://openalex.org/W1595672120","https://openalex.org/W4230999561","https://openalex.org/W2405661381"],"abstract_inverted_index":{"In":[0,100],"the":[1,5,23,37,56,69,119],"last":[2],"three":[3],"years,":[4],"largest":[6],"dense":[7],"deep":[8],"learning":[9],"models":[10,52,103],"have":[11],"grown":[12,28],"over":[13],"1000x":[14],"to":[15,33,53,68,80,122],"reach":[16,95],"hundreds":[17],"of":[18,20,60,94,110],"billions":[19],"parameters,":[21],"while":[22],"GPU":[24,58,70],"memory":[25,59,71],"has":[26,42],"only":[27],"by":[29],"5x":[30],"(16":[31],"GB":[32],"80":[34],"GB).":[35],"Therefore,":[36],"growth":[38],"in":[39,55],"model":[40,85],"scale":[41,106],"been":[43],"supported":[44],"primarily":[45],"though":[46],"system":[47],"innovations":[48],"that":[49,105,113],"allow":[50],"large":[51],"fit":[54,81],"aggregate":[57],"multiple":[61],"GPUs.":[62],"However,":[63],"we":[64],"are":[65,91],"getting":[66],"close":[67],"wall.":[72],"It":[73],"requires":[74,107],"800":[75],"NVIDIA":[76],"V100":[77],"GPUs":[78],"just":[79],"a":[82,115],"trillion":[83],"parameter":[84],"for":[86,96],"training,":[87],"and":[88],"such":[89],"clusters":[90],"simply":[92],"out":[93],"most":[97],"data":[98,120],"scientists.":[99],"addition,":[101],"training":[102],"at":[104],"complex":[108],"combinations":[109],"parallelism":[111],"techniques":[112],"puts":[114],"big":[116],"burden":[117],"on":[118],"scientists":[121],"refactor":[123],"their":[124],"model.":[125]},"counts_by_year":[{"year":2026,"cited_by_count":22},{"year":2025,"cited_by_count":69},{"year":2024,"cited_by_count":66},{"year":2023,"cited_by_count":46},{"year":2022,"cited_by_count":16}],"updated_date":"2026-06-23T13:55:30.953635","created_date":"2021-10-25T00:00:00"}