{"id":"https://openalex.org/W7138358901","doi":"https://doi.org/10.1609/aaai.v40i32.39901","title":"TawPipe: Topology-Aware Weight Pipeline Parallelism for Accelerating Long-Context Large Models Training","display_name":"TawPipe: Topology-Aware Weight Pipeline Parallelism for Accelerating Long-Context Large Models Training","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7138358901","doi":"https://doi.org/10.1609/aaai.v40i32.39901"},"language":null,"primary_location":{"id":"doi:10.1609/aaai.v40i32.39901","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i32.39901","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/39901/43862","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/39901/43862","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100834970","display_name":"Houming Wu","orcid":null},"institutions":[{"id":"https://openalex.org/I55712492","display_name":"Zhejiang University of Technology","ror":"https://ror.org/02djqfd08","country_code":"CN","type":"education","lineage":["https://openalex.org/I55712492"]},{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Houming Wu","raw_affiliation_strings":["State Key Laboratory of Blockchain and Data Security, Zhejiang University\nCollege of Computer Science and Technology, Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Blockchain and Data Security, Zhejiang University\nCollege of Computer Science and Technology, Zhejiang University","institution_ids":["https://openalex.org/I55712492","https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5129736312","display_name":"Ling Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I55712492","display_name":"Zhejiang University of Technology","ror":"https://ror.org/02djqfd08","country_code":"CN","type":"education","lineage":["https://openalex.org/I55712492"]},{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ling Chen","raw_affiliation_strings":["State Key Laboratory of Blockchain and Data Security, Zhejiang University\nCollege of Computer Science and Technology, Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Blockchain and Data Security, Zhejiang University\nCollege of Computer Science and Technology, Zhejiang University","institution_ids":["https://openalex.org/I55712492","https://openalex.org/I76130692"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.27079155,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"32","first_page":"26894","last_page":"26902"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.20270000398159027,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.20270000398159027,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.13040000200271606,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.06949999928474426,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7608000040054321},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.7504000067710876},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.6478999853134155},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.5666000247001648},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5318999886512756},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.49540001153945923},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.4936000108718872},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.445499986410141},{"id":"https://openalex.org/keywords/bandwidth","display_name":"Bandwidth (computing)","score":0.4415999948978424}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.833899974822998},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7608000040054321},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.7504000067710876},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.6478999853134155},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6388999819755554},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5875999927520752},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.5666000247001648},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5318999886512756},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.49540001153945923},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.4936000108718872},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.445499986410141},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.4415999948978424},{"id":"https://openalex.org/C62611344","wikidata":"https://www.wikidata.org/wiki/Q1062658","display_name":"Node (physics)","level":2,"score":0.4372999966144562},{"id":"https://openalex.org/C158156997","wikidata":"https://www.wikidata.org/wiki/Q1416645","display_name":"Models of communication","level":2,"score":0.42239999771118164},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.4163999855518341},{"id":"https://openalex.org/C2776834041","wikidata":"https://www.wikidata.org/wiki/Q25346349","display_name":"Execution model","level":2,"score":0.3736000061035156},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.37299999594688416},{"id":"https://openalex.org/C199845137","wikidata":"https://www.wikidata.org/wiki/Q145490","display_name":"Network topology","level":2,"score":0.36329999566078186},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.3327000141143799},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.33000001311302185},{"id":"https://openalex.org/C193702766","wikidata":"https://www.wikidata.org/wiki/Q1414548","display_name":"Concurrency","level":2,"score":0.3027999997138977},{"id":"https://openalex.org/C31170391","wikidata":"https://www.wikidata.org/wiki/Q188619","display_name":"Hierarchy","level":2,"score":0.2833000123500824},{"id":"https://openalex.org/C557945733","wikidata":"https://www.wikidata.org/wiki/Q389772","display_name":"Data transmission","level":2,"score":0.2768000066280365},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.27239999175071716},{"id":"https://openalex.org/C138959212","wikidata":"https://www.wikidata.org/wiki/Q1806783","display_name":"Load balancing (electrical power)","level":3,"score":0.2639000117778778},{"id":"https://openalex.org/C91481028","wikidata":"https://www.wikidata.org/wiki/Q1054686","display_name":"Distributed memory","level":3,"score":0.25760000944137573},{"id":"https://openalex.org/C106515295","wikidata":"https://www.wikidata.org/wiki/Q26806595","display_name":"Parallel processing","level":2,"score":0.2551000118255615}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1609/aaai.v40i32.39901","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i32.39901","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/39901/43862","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i32.39901","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i32.39901","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/39901/43862","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7138358901.pdf","grobid_xml":"https://content.openalex.org/works/W7138358901.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Training":[0],"large":[1],"language":[2],"models":[3,24,155],"(LLMs)":[4],"is":[5],"fundamentally":[6],"constrained":[7],"by":[8,22,50],"limited":[9],"device":[10,103],"memory":[11,20],"and":[12,64,96,110,115,162],"costly":[13],"inter-device":[14],"communication.":[15],"Although":[16],"pipeline":[17,72],"parallelism":[18,133],"alleviates":[19],"pressure":[21],"partitioning":[23],"across":[25],"devices,":[26],"it":[27],"incurs":[28],"activation":[29],"communication":[30,83,118,138],"overhead":[31],"that":[32,157],"scales":[33],"linearly":[34],"with":[35,119,153],"sequence":[36],"length,":[37],"limiting":[38],"efficiency":[39],"in":[40,78,129],"long-context":[41],"training.":[42],"Recent":[43],"weight-passing":[44],"approaches":[45],"(e.g.,":[46],"WeiPipe)":[47],"mitigate":[48],"this":[49],"transmitting":[51],"model":[52,108],"weights":[53,109],"instead":[54],"of":[55,107],"activations,":[56],"but":[57],"suffer":[58],"from":[59],"redundant":[60,113],"peer-to-peer":[61],"(P2P)":[62],"transfers":[63],"underutilized":[65],"intra-node":[66,94],"bandwidth.":[67],"We":[68],"propose":[69],"TawPipe\u2014topology-aware":[70],"weight":[71],"parallelism,":[73],"which":[74],"exploits":[75],"hierarchical":[76],"bandwidth":[77],"distributed":[79],"clusters":[80],"for":[81],"improved":[82],"efficiency.":[84],"TawPipe:":[85],"(i)":[86],"groups":[87],"devices":[88],"based":[89],"on":[90,148],"topology":[91],"to":[92,121,150,165],"optimize":[93],"collective":[95,126],"inter-node":[97],"P2P":[98],"communication;":[99],"(ii)":[100],"assigns":[101],"each":[102],"a":[104],"fixed":[105],"shard":[106],"gradients,":[111],"avoiding":[112],"transfers;":[114],"(iii)":[116],"overlaps":[117],"computation":[120],"hide":[122],"latency.":[123],"Unlike":[124],"global":[125],"operations":[127],"used":[128],"fully":[130],"sharded":[131],"data":[132],"(FSDP),":[134],"TawPipe":[135,158],"confines":[136],"most":[137],"within":[139],"node":[140],"boundaries,":[141],"significantly":[142],"reducing":[143],"cross-node":[144],"traffic.":[145],"Extensive":[146],"experiments":[147],"up":[149],"24":[151],"GPUs":[152],"LLaMA\u2011style":[154],"show":[156],"achieves":[159],"superior":[160],"throughput":[161],"scalability":[163],"compared":[164],"state-of-the-art":[166],"baselines.":[167]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-03-18T00:00:00"}