{"id":"https://openalex.org/W4417403573","doi":"https://doi.org/10.1109/pact65351.2025.00014","title":"SPipe: Hybrid GPU and CPU Pipeline for Training LLMs under Memory Pressure","display_name":"SPipe: Hybrid GPU and CPU Pipeline for Training LLMs under Memory Pressure","publication_year":2025,"publication_date":"2025-11-03","ids":{"openalex":"https://openalex.org/W4417403573","doi":"https://doi.org/10.1109/pact65351.2025.00014"},"language":null,"primary_location":{"id":"doi:10.1109/pact65351.2025.00014","is_oa":false,"landing_page_url":"https://doi.org/10.1109/pact65351.2025.00014","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 34th International Conference on Parallel Architectures and Compilation Techniques (PACT)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032058183","display_name":"Junyeol Ryu","orcid":"https://orcid.org/0009-0004-8788-7405"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":true,"raw_author_name":"Junyeol Ryu","raw_affiliation_strings":["Seoul National University,Dept. of Computer Science and Engineering,Seoul,South Korea"],"affiliations":[{"raw_affiliation_string":"Seoul National University,Dept. of Computer Science and Engineering,Seoul,South Korea","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100712654","display_name":"Yujin Jeong","orcid":"https://orcid.org/0009-0004-2715-5501"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Yujin Jeong","raw_affiliation_strings":["Seoul National University,Dept. of Computer Science and Engineering,Seoul,South Korea"],"affiliations":[{"raw_affiliation_string":"Seoul National University,Dept. of Computer Science and Engineering,Seoul,South Korea","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083539980","display_name":"Daeyoung Park","orcid":"https://orcid.org/0000-0001-8573-3526"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Daeyoung Park","raw_affiliation_strings":["Seoul National University,Dept. of Computer Science and Engineering,Seoul,South Korea"],"affiliations":[{"raw_affiliation_string":"Seoul National University,Dept. of Computer Science and Engineering,Seoul,South Korea","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006733433","display_name":"Jin-Pyo Kim","orcid":"https://orcid.org/0000-0002-4370-1195"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jinpyo Kim","raw_affiliation_strings":["Seoul National University,Dept. of Computer Science and Engineering,Seoul,South Korea"],"affiliations":[{"raw_affiliation_string":"Seoul National University,Dept. of Computer Science and Engineering,Seoul,South Korea","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057343217","display_name":"Heehoon Kim","orcid":"https://orcid.org/0000-0002-5392-4413"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Heehoon Kim","raw_affiliation_strings":["Seoul National University,Dept. of Computer Science and Engineering,Seoul,South Korea"],"affiliations":[{"raw_affiliation_string":"Seoul National University,Dept. of Computer Science and Engineering,Seoul,South Korea","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100767182","display_name":"Jaejin Lee","orcid":"https://orcid.org/0000-0003-4638-8170"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jaejin Lee","raw_affiliation_strings":["Seoul National University,Dept. of Computer Science and Engineering,Seoul,South Korea"],"affiliations":[{"raw_affiliation_string":"Seoul National University,Dept. of Computer Science and Engineering,Seoul,South Korea","institution_ids":["https://openalex.org/I139264467"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5032058183"],"corresponding_institution_ids":["https://openalex.org/I139264467"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.21119083,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"14","last_page":"29"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2565999925136566,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2565999925136566,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.10750000178813934,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09830000251531601,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.8450999855995178},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.6018999814987183},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.5180000066757202},{"id":"https://openalex.org/keywords/pipeline-transport","display_name":"Pipeline transport","score":0.454800009727478},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.43779999017715454},{"id":"https://openalex.org/keywords/central-processing-unit","display_name":"Central processing unit","score":0.34599998593330383},{"id":"https://openalex.org/keywords/data-parallelism","display_name":"Data parallelism","score":0.3425999879837036}],"concepts":[{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.8450999855995178},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8323000073432922},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.6018999814987183},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5939000248908997},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.5180000066757202},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.454800009727478},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.43779999017715454},{"id":"https://openalex.org/C49154492","wikidata":"https://www.wikidata.org/wiki/Q5300","display_name":"Central processing unit","level":2,"score":0.34599998593330383},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.3425999879837036},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.335999995470047},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.32280001044273376},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.3208000063896179},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.31209999322891235},{"id":"https://openalex.org/C83283714","wikidata":"https://www.wikidata.org/wiki/Q121117","display_name":"Supercomputer","level":2,"score":0.289900004863739},{"id":"https://openalex.org/C50630238","wikidata":"https://www.wikidata.org/wiki/Q971505","display_name":"General-purpose computing on graphics processing units","level":3,"score":0.2759999930858612},{"id":"https://openalex.org/C557945733","wikidata":"https://www.wikidata.org/wiki/Q389772","display_name":"Data transmission","level":2,"score":0.27079999446868896},{"id":"https://openalex.org/C106515295","wikidata":"https://www.wikidata.org/wiki/Q26806595","display_name":"Parallel processing","level":2,"score":0.2669999897480011},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.2655999958515167},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.25459998846054077}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/pact65351.2025.00014","is_oa":false,"landing_page_url":"https://doi.org/10.1109/pact65351.2025.00014","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 34th International Conference on Parallel Architectures and Compilation Techniques (PACT)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321292","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542"},{"id":"https://openalex.org/F4320321408","display_name":"Ministry of Education","ror":"https://ror.org/01p262204"},{"id":"https://openalex.org/F4320322120","display_name":"National Research Foundation of Korea","ror":"https://ror.org/013aysd81"},{"id":"https://openalex.org/F4320326288","display_name":"K2","ror":null},{"id":"https://openalex.org/F4320338440","display_name":"HORIZON EUROPE Health","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W2489529491","https://openalex.org/W2786680212","https://openalex.org/W2891993230","https://openalex.org/W2962747323","https://openalex.org/W2969388332","https://openalex.org/W2972087877","https://openalex.org/W2991040477","https://openalex.org/W3010830594","https://openalex.org/W3012479151","https://openalex.org/W3012514909","https://openalex.org/W3081168214","https://openalex.org/W3120480394","https://openalex.org/W3129831491","https://openalex.org/W3132107458","https://openalex.org/W3157864729","https://openalex.org/W3206832494","https://openalex.org/W4220741164","https://openalex.org/W4318541673","https://openalex.org/W4386348101","https://openalex.org/W4386707654","https://openalex.org/W4386763939","https://openalex.org/W4391215636","https://openalex.org/W4398796293","https://openalex.org/W4399794472","https://openalex.org/W4404340628"],"related_works":[],"abstract_inverted_index":{"Training":[0],"large":[1],"language":[2],"models":[3],"(LLMs)":[4],"with":[5,107],"limited":[6],"computing":[7],"resources":[8,48],"is":[9],"challenging":[10],"because":[11],"of":[12,67,104],"their":[13],"immense":[14],"memory":[15,33],"space":[16],"requirements.":[17],"In":[18,53],"this":[19,54],"paper,":[20],"we":[21,28],"specifically":[22],"focus":[23],"on":[24],"the":[25,51,75,118],"scenarios":[26],"where":[27],"have":[29],"insufficient":[30],"aggregate":[31],"GPU":[32,60,71],"to":[34,49,73,85],"store":[35],"all":[36,46],"model":[37],"states":[38],"but":[39],"explore":[40],"pipeline":[41,72,79,84],"parallelism":[42,80],"and":[43,61,81,90,96],"offloading":[44,94],"across":[45],"system":[47],"train":[50],"model.":[52],"context,":[55],"SPipe":[56,100,116],"presents":[57],"a":[58,70,82],"hybrid":[59],"CPU":[62,91],"pipelining":[63],"mechanism":[64],"that":[65,115],"consists":[66],"two":[68],"pipelines:":[69],"reduce":[74],"bubbles":[76],"in":[77,93,110],"conventional":[78],"GPU-CPU":[83],"alleviate":[86],"data":[87,95],"transfer":[88],"overhead":[89],"bottlenecks":[92],"computation.":[97],"We":[98],"evaluate":[99],"for":[101],"training":[102],"LLMs":[103],"various":[105],"sizes":[106],"diverse":[108],"configurations":[109],"practice.":[111],"The":[112],"result":[113],"indicates":[114],"outperforms":[117],"state-of-the-art":[119],"by":[120],"$1.26":[121],"\\times$.":[122]},"counts_by_year":[],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-12-16T00:00:00"}