{"id":"https://openalex.org/W4403211358","doi":"https://doi.org/10.1109/ccgrid59990.2024.00031","title":"Accelerating Large Language Model Training with Hybrid GPU-based Compression","display_name":"Accelerating Large Language Model Training with Hybrid GPU-based Compression","publication_year":2024,"publication_date":"2024-05-06","ids":{"openalex":"https://openalex.org/W4403211358","doi":"https://doi.org/10.1109/ccgrid59990.2024.00031"},"language":"en","primary_location":{"id":"doi:10.1109/ccgrid59990.2024.00031","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ccgrid59990.2024.00031","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE 24th International Symposium on Cluster, Cloud and Internet Computing (CCGrid)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101847978","display_name":"Lang Xu","orcid":"https://orcid.org/0009-0000-6307-4566"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lang Xu","raw_affiliation_strings":["The Ohio State University,Columbus,Ohio"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Columbus,Ohio","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015153122","display_name":"Quentin Anthony","orcid":"https://orcid.org/0000-0002-6823-9080"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Quentin Anthony","raw_affiliation_strings":["The Ohio State University,Columbus,Ohio"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Columbus,Ohio","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028296912","display_name":"Qinghua Zhou","orcid":"https://orcid.org/0009-0001-8490-7990"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qinghua Zhou","raw_affiliation_strings":["The Ohio State University,Columbus,Ohio"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Columbus,Ohio","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061967476","display_name":"Nawras Alnaasan","orcid":"https://orcid.org/0000-0002-3638-4144"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nawras Alnaasan","raw_affiliation_strings":["The Ohio State University,Columbus,Ohio"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Columbus,Ohio","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104772578","display_name":"Radha Gulhane","orcid":"https://orcid.org/0009-0009-2591-1082"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Radha Gulhane","raw_affiliation_strings":["The Ohio State University,Columbus,Ohio"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Columbus,Ohio","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078128277","display_name":"Aamir Shafi","orcid":"https://orcid.org/0000-0002-1924-2769"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aamir Shafi","raw_affiliation_strings":["The Ohio State University,Columbus,Ohio"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Columbus,Ohio","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034293705","display_name":"Hari Subramoni","orcid":"https://orcid.org/0000-0002-1200-2754"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hari Subramoni","raw_affiliation_strings":["The Ohio State University,Columbus,Ohio"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Columbus,Ohio","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024879682","display_name":"Dhabaleswar K. Panda","orcid":"https://orcid.org/0000-0002-0356-1781"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dhabaleswar K. DK Panda","raw_affiliation_strings":["The Ohio State University,Columbus,Ohio"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Columbus,Ohio","institution_ids":["https://openalex.org/I52357470"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I52357470"],"apc_list":null,"apc_paid":null,"fwci":2.8448,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.92642312,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"196","last_page":"205"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9908999800682068,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.957099974155426,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7856261730194092},{"id":"https://openalex.org/keywords/compression","display_name":"Compression (physics)","score":0.4953135550022125},{"id":"https://openalex.org/keywords/data-compression","display_name":"Data compression","score":0.42076927423477173},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.37501442432403564},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.35385721921920776},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2671862840652466},{"id":"https://openalex.org/keywords/composite-material","display_name":"Composite material","score":0.06751367449760437},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.05478936433792114}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7856261730194092},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.4953135550022125},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.42076927423477173},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.37501442432403564},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.35385721921920776},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2671862840652466},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.06751367449760437},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.05478936433792114}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ccgrid59990.2024.00031","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ccgrid59990.2024.00031","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE 24th International Symposium on Cluster, Cloud and Internet Computing (CCGrid)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W1902814702","https://openalex.org/W2052440657","https://openalex.org/W2964339011","https://openalex.org/W2972087877","https://openalex.org/W3043494828","https://openalex.org/W3046490292","https://openalex.org/W3081168214","https://openalex.org/W3132977829","https://openalex.org/W3177196615","https://openalex.org/W3177445289","https://openalex.org/W3205803342","https://openalex.org/W4285105236","https://openalex.org/W4285225959","https://openalex.org/W4285297247","https://openalex.org/W4367147749","https://openalex.org/W4383749754","https://openalex.org/W4384705436","https://openalex.org/W4384705453","https://openalex.org/W6745245109","https://openalex.org/W6746839373","https://openalex.org/W6748019269","https://openalex.org/W6755207826","https://openalex.org/W6756718674","https://openalex.org/W6767997687","https://openalex.org/W6768317493","https://openalex.org/W6772383348","https://openalex.org/W6788175385","https://openalex.org/W6790665546","https://openalex.org/W6802298259","https://openalex.org/W6810296985","https://openalex.org/W6850625674","https://openalex.org/W6852686493","https://openalex.org/W6853287796"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2612632602","https://openalex.org/W2321805087"],"abstract_inverted_index":{"Data":[0],"Parallelism":[1,4,8],"(DP),":[2],"Tensor":[3],"(TP),":[5],"and":[6,19,37,41,65,97,121,130,160,176,209,214,231],"Pipeline":[7],"(PP)":[9],"are":[10],"the":[11,81,88,108,146,178,217],"three":[12],"strategies":[13],"widely":[14],"adopted":[15],"to":[16,34,60,103,201],"enable":[17],"fast":[18],"efficient":[20],"Large":[21],"Language":[22],"Model":[23],"(LLM)":[24],"training.":[25,140],"However,":[26],"these":[27],"approaches":[28],"rely":[29],"on":[30,107,191],"data-intensive":[31],"communication":[32],"routines":[33],"collect,":[35],"aggregate,":[36],"re-distribute":[38],"gradients,":[39],"activations,":[40,206],"other":[42],"important":[43],"model":[44,210],"information,":[45],"which":[46],"pose":[47],"significant":[48],"overhead.":[49],"Co-designed":[50],"with":[51],"GPU-based":[52],"compression":[53,116,170,179,190,200,220],"libraries,":[54],"MPI":[55,85],"libraries":[56],"have":[57],"been":[58],"proven":[59],"reduce":[61],"message":[62],"size":[63],"significantly,":[64],"leverage":[66],"interconnect":[67],"bandwidth,":[68],"thus":[69,156],"increasing":[70],"training":[71,93,164],"efficiency":[72],"while":[73,204,239],"maintaining":[74],"acceptable":[75],"accuracy.In":[76],"this":[77],"work,":[78],"we":[79,112,167,187,222],"investigate":[80],"efficacy":[82],"of":[83,90],"compression-assisted":[84],"collectives":[86,120],"under":[87],"context":[89],"distributed":[91],"LLM":[92],"using":[94],"3D":[95],"parallelism":[96,154],"ZeRO":[98],"optimizations.":[99],"We":[100,197],"scaled":[101],"up":[102],"192":[104],"V100":[105],"GPUs":[106],"Lassen":[109],"supercomputer.":[110],"First,":[111],"enabled":[113],"a":[114,123,131,143,224,232],"na\u00efve":[115],"scheme":[117],"across":[118],"all":[119],"observed":[122],"22.5%":[124],"increase":[125,133,226,234],"in":[126,134,152,163,212,227,235],"TFLOPS":[127,228],"per":[128,136,229,237],"GPU":[129,230],"23.6%":[132],"samples":[135,236],"second":[137,238],"for":[138],"GPT-NeoX-20B":[139],"Nonetheless,":[141],"such":[142],"strategy":[144],"ignores":[145],"sparsity":[147],"discrepancy":[148],"among":[149],"messages":[150],"communicated":[151],"each":[153,173],"degree,":[155],"introducing":[157],"more":[158],"errors":[159],"causing":[161],"degradation":[162],"loss.":[165],"Therefore,":[166],"incorporated":[168],"hybrid":[169,219],"settings":[171],"toward":[172],"parallel":[174],"dimension":[175],"adjusted":[177,218],"intensity":[180],"accordingly.":[181],"Given":[182],"their":[183],"low-rank":[184],"structure":[185],"[1],":[186],"apply":[188],"aggressive":[189],"gradients":[192],"when":[193],"performing":[194],"DP":[195],"All-reduce.":[196],"adopt":[198],"milder":[199],"preserve":[202],"precision":[203],"communicating":[205],"optimizer":[207],"states,":[208],"parameters":[211],"TP":[213],"PP.":[215],"Using":[216],"scheme,":[221],"demonstrate":[223],"17.3%":[225],"12.7%":[233],"reaching":[240],"baseline":[241],"loss":[242],"convergence.*":[243]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}