{"id":"https://openalex.org/W4412945374","doi":"https://doi.org/10.18653/v1/2025.acl-long.525","title":"APB: Accelerating Distributed Long-Context Inference by Passing Compressed Context Blocks across GPUs","display_name":"APB: Accelerating Distributed Long-Context Inference by Passing Compressed Context Blocks across GPUs","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412945374","doi":"https://doi.org/10.18653/v1/2025.acl-long.525"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.acl-long.525","doi":"https://doi.org/10.18653/v1/2025.acl-long.525","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.525","pdf_url":"https://aclanthology.org/2025.acl-long.525.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://aclanthology.org/2025.acl-long.525.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103140704","display_name":"Yuxiang Huang","orcid":"https://orcid.org/0000-0002-4380-2447","observed_orcids":["https://orcid.org/0000-0002-4380-2447","https://orcid.org/0009-0004-7629-1454","https://orcid.org/0009-0009-3382-8262"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuxiang Huang","raw_affiliation_strings":["NLP Group , DCST, IAI , BNRIST , Tsinghua University , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Group , DCST, IAI , BNRIST , Tsinghua University , Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102743371","display_name":"Mingye Li","orcid":"https://orcid.org/0000-0002-1126-3178","observed_orcids":["https://orcid.org/0000-0002-1126-3178"]},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingye Li","raw_affiliation_strings":["Department of CS&T , Central South University , Changsha , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of CS&T , Central South University , Changsha , China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101224117","display_name":"Xu Han","orcid":"https://orcid.org/0009-0006-8542-8509","observed_orcids":["https://orcid.org/0009-0006-8542-8509"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xu Han","raw_affiliation_strings":["NLP Group , DCST, IAI , BNRIST , Tsinghua University , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Group , DCST, IAI , BNRIST , Tsinghua University , Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014699953","display_name":"Chaojun Xiao","orcid":"https://orcid.org/0000-0001-6039-0942","observed_orcids":["https://orcid.org/0000-0001-6039-0942","https://orcid.org/0009-0003-4475-0946"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chaojun Xiao","raw_affiliation_strings":["NLP Group , DCST, IAI , BNRIST , Tsinghua University , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Group , DCST, IAI , BNRIST , Tsinghua University , Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035433961","display_name":"Weilin Zhao","orcid":"https://orcid.org/0000-0001-9352-6252","observed_orcids":["https://orcid.org/0000-0001-9352-6252","https://orcid.org/0000-0001-8016-1952"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weilin Zhao","raw_affiliation_strings":["NLP Group , DCST, IAI , BNRIST , Tsinghua University , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Group , DCST, IAI , BNRIST , Tsinghua University , Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058302872","display_name":"Ao Sun","orcid":"https://orcid.org/0009-0001-0219-4780","observed_orcids":["https://orcid.org/0009-0001-0219-4780"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun Ao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Hao Zhou","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Zhou","raw_affiliation_strings":["Pattern Recognition Center , WeChat AI , Tencent Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pattern Recognition Center , WeChat AI , Tencent Inc","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017312552","display_name":"Jie Zhou","orcid":"https://orcid.org/0009-0009-5198-6782","observed_orcids":["https://orcid.org/0009-0009-5198-6782"]},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jie Zhou","raw_affiliation_strings":["Pattern Recognition Center , WeChat AI , Tencent Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pattern Recognition Center , WeChat AI , Tencent Inc","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100320720","display_name":"Zhiyuan Liu","orcid":"https://orcid.org/0000-0002-4943-6572","observed_orcids":["https://orcid.org/0000-0002-4943-6572","https://orcid.org/0009-0002-2240-1553","https://orcid.org/0009-0007-4866-3670","https://orcid.org/0009-0008-5881-6615"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiyuan Liu","raw_affiliation_strings":["NLP Group , DCST, IAI , BNRIST , Tsinghua University , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Group , DCST, IAI , BNRIST , Tsinghua University , Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046448314","display_name":"Maosong Sun","orcid":"https://orcid.org/0000-0002-6011-6115","observed_orcids":["https://orcid.org/0000-0002-6011-6115"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Maosong Sun","raw_affiliation_strings":["NLP Group , DCST, IAI , BNRIST , Tsinghua University , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP Group , DCST, IAI , BNRIST , Tsinghua University , Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"10708","last_page":"10727"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.4672999978065491,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.4672999978065491,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.24240000545978546,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.19480000436306,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/long-context-inference","display_name":"long-context inference","score":0.996999979019165},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8980000019073486},{"id":"https://openalex.org/keywords/flashattention","display_name":"FlashAttention","score":0.5289999842643738},{"id":"https://openalex.org/keywords/distributed-inference","display_name":"distributed inference","score":0.6039999723434448},{"id":"https://openalex.org/keywords/gpu-parallelization","display_name":"GPU parallelization","score":0.6660000085830688}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7969884872436523},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.7182133197784424},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6651994585990906},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6513550281524658},{"id":"https://openalex.org/C854659","wikidata":"https://www.wikidata.org/wiki/Q1859284","display_name":"Message passing","level":2,"score":0.53659987449646},{"id":"https://openalex.org/C459310","wikidata":"https://www.wikidata.org/wiki/Q117801","display_name":"Computational science","level":1,"score":0.34332090616226196},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.13764524459838867},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.10284125804901123},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2025.acl-long.525","doi":"https://doi.org/10.18653/v1/2025.acl-long.525","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.525","pdf_url":"https://aclanthology.org/2025.acl-long.525.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2502.12085","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.12085","pdf_url":"https://arxiv.org/pdf/2502.12085","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.12085","doi":"https://doi.org/10.48550/arxiv.2502.12085","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.12085","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.acl-long.525","doi":"https://doi.org/10.18653/v1/2025.acl-long.525","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.525","pdf_url":"https://aclanthology.org/2025.acl-long.525.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320322392","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412945374.pdf","grobid_xml":"https://content.openalex.org/works/W4412945374.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W3015468748","https://openalex.org/W4388685466","https://openalex.org/W4392873765","https://openalex.org/W4399452094","https://openalex.org/W4400024704","https://openalex.org/W4400611829","https://openalex.org/W4401306886","https://openalex.org/W4402671875","https://openalex.org/W4403580165","https://openalex.org/W4404340748","https://openalex.org/W4404649984","https://openalex.org/W4404988499","https://openalex.org/W4405434119","https://openalex.org/W4405903187","https://openalex.org/W4406465766","https://openalex.org/W4415797154","https://openalex.org/W6941243218","https://openalex.org/W7128942817"],"related_works":["https://openalex.org/W4404399538","https://openalex.org/W4415797534","https://openalex.org/W4416930301","https://openalex.org/W4408355184","https://openalex.org/W7160632811","https://openalex.org/W7203866566","https://openalex.org/W4416019938","https://openalex.org/W4415193952","https://openalex.org/W7134277865","https://openalex.org/W7131416759"],"abstract_inverted_index":{"While":[0],"long-context":[1,48,61],"inference":[2,37,62,96],"is":[3],"crucial":[4],"for":[5,84],"advancing":[6],"large":[7],"language":[8],"model":[9],"(LLM)":[10],"applications,":[11],"its":[12],"prefill":[13,71],"speed":[14,72,97],"remains":[15],"a":[16,51,81,89,94,106],"significant":[17],"bottleneck.Current":[18],"approaches,":[19],"including":[20],"sequence":[21,90],"parallelism":[22,78,91,118],"strategies":[23],"and":[24,46,76,117,127,133,145],"compute":[25,75],"reduction":[26],"through":[27],"approximate":[28,67],"attention":[29,68],"mechanisms,":[30],"still":[31],"fall":[32],"short":[33],"of":[34,122,148],"delivering":[35],"optimal":[36],"efficiency.This":[38],"hinders":[39],"scaling":[40],"the":[41,143],"inputs":[42],"to":[43,69,124],"longer":[44],"sequences":[45],"processing":[47],"queries":[49],"in":[50,150],"timely":[52],"manner.To":[53],"address":[54],"this,":[55],"we":[56],"introduce":[57],"APB,":[58],"an":[59],"efficient":[60],"framework":[63],"that":[64],"leverages":[65],"multi-host":[66],"enhance":[70],"by":[73,104],"reducing":[74],"enhancing":[77],"simultaneously.APB":[79],"introduces":[80],"communication":[82],"mechanism":[83],"essential":[85],"key-value":[86],"pairs":[87],"within":[88],"framework,":[92],"enabling":[93],"faster":[95],"while":[98],"maintaining":[99],"task":[100,139],"performance.We":[101],"implement":[102],"APB":[103,149],"incorporating":[105],"tailored":[107],"FLASHATTN":[108],"kernel":[109],"alongside":[110],"optimized":[111],"distribution":[112],"strategies,":[113],"supporting":[114],"diverse":[115],"models":[116],"configurations.APB":[119],"achieves":[120],"speedups":[121],"up":[123],"9.2\u00d7,":[125],"4.2\u00d7,":[126],"1.6\u00d7":[128],"compared":[129],"with":[130],"FLASHATTN,":[131],"RINGATTN,":[132],"STARATTN,":[134],"respectively,":[135],"without":[136],"any":[137],"observable":[138],"performance":[140],"degradation.We":[141],"provide":[142],"implementation":[144],"experiment":[146],"code":[147],"https://github.com/thunlp/APB.*":[151],"indicates":[152],"equal":[153],"contribution.":[154]},"counts_by_year":[],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2025-10-10T00:00:00"}