{"id":"https://openalex.org/W7166649151","doi":"https://doi.org/10.48550/arxiv.2606.29986","title":"HBM Is Not All You Need: Efficient Disaggregated LLM Serving across Memory-heterogeneous Accelerators","display_name":"HBM Is Not All You Need: Efficient Disaggregated LLM Serving across Memory-heterogeneous Accelerators","publication_year":2026,"publication_date":"2026-06-29","ids":{"openalex":"https://openalex.org/W7166649151","doi":"https://doi.org/10.48550/arxiv.2606.29986"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.29986","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29986","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.29986","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139641599","display_name":"Zhixiang Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Zhixiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139685630","display_name":"Yun Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121259042","display_name":"James Yen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yen, James","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049740011","display_name":"Mingyuan Xia","orcid":"https://orcid.org/0000-0001-5899-0295"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Mingyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139696570","display_name":"Zhengwei Qi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi, Zhengwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.6545000076293945,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.6545000076293945,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.16009999811649323,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.05959999933838844,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/goodput","display_name":"Goodput","score":0.8241000175476074},{"id":"https://openalex.org/keywords/byte","display_name":"Byte","score":0.6460999846458435},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6036999821662903},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.539900004863739},{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.5008000135421753},{"id":"https://openalex.org/keywords/bandwidth","display_name":"Bandwidth (computing)","score":0.498199999332428},{"id":"https://openalex.org/keywords/idle","display_name":"Idle","score":0.4523000121116638},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.3449999988079071}],"concepts":[{"id":"https://openalex.org/C94022561","wikidata":"https://www.wikidata.org/wiki/Q1172393","display_name":"Goodput","level":4,"score":0.8241000175476074},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8095999956130981},{"id":"https://openalex.org/C43364308","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Byte","level":2,"score":0.6460999846458435},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6036999821662903},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.539900004863739},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.5008000135421753},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.498199999332428},{"id":"https://openalex.org/C16320812","wikidata":"https://www.wikidata.org/wiki/Q1812200","display_name":"Idle","level":2,"score":0.4523000121116638},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.3449999988079071},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.34310001134872437},{"id":"https://openalex.org/C165005293","wikidata":"https://www.wikidata.org/wiki/Q1074500","display_name":"Chip","level":2,"score":0.3345000147819519},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.33149999380111694},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.321399986743927},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3046000003814697},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2827000021934509},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.2777000069618225},{"id":"https://openalex.org/C149810388","wikidata":"https://www.wikidata.org/wiki/Q5374873","display_name":"Emulation","level":2,"score":0.27570000290870667},{"id":"https://openalex.org/C99138194","wikidata":"https://www.wikidata.org/wiki/Q183427","display_name":"Hash function","level":2,"score":0.27390000224113464},{"id":"https://openalex.org/C95203288","wikidata":"https://www.wikidata.org/wiki/Q221682","display_name":"Semaphore","level":2,"score":0.2736000120639801},{"id":"https://openalex.org/C3913047","wikidata":"https://www.wikidata.org/wiki/Q1956265","display_name":"sync","level":3,"score":0.26980000734329224},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2646999955177307},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.2574999928474426},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.25690001249313354}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.29986","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29986","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.29986","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29986","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"LLM":[0,36],"inference":[1],"comprises":[2],"a":[3,8,79,93,101,108,145],"compute-bound":[4],"prefill":[5,46,117,137,157],"phase":[6,75],"and":[7,12,100,162,170,182,190,205],"memory-bound":[9],"decode":[10,122,140,176],"phase,":[11],"recent":[13],"systems":[14],"disaggregate":[15],"them":[16,172],"onto":[17],"separate":[18],"hardware.":[19],"Yet":[20],"today's":[21],"datacenter":[22],"GPUs":[23,49,120],"rely":[24],"on":[25,174,213],"costly":[26],"HBM":[27,183],"whose":[28],"bandwidth":[29,181],"sits":[30],"almost":[31],"entirely":[32],"idle":[33],"during":[34],"prefill.":[35],"serving":[37,65,111],"across":[38],"memory-heterogeneous":[39],"accelerators":[40,44,115],"(MemHA)":[41],"pairs":[42],"GDDR-based":[43,114],"for":[45,50,73,90,116,121,135],"with":[47,118,155,209],"HBM-based":[48,119],"decode,":[51],"promising":[52],"lower":[53],"cost":[54],"without":[55],"sacrificing":[56],"performance.":[57],"Pushed":[58],"to":[59,158,178,197],"its":[60],"most":[61],"economical":[62],"form,":[63],"MemHA":[64],"is":[66],"inherently":[67],"cross-vendor,":[68],"since":[69],"the":[70,175],"best-suited":[71],"chip":[72],"each":[74,150],"may":[76],"come":[77],"from":[78],"different":[80],"vendor.":[81],"This":[82],"breaks":[83],"two":[84],"assumptions":[85],"that":[86,148],"single-vendor":[87],"disaggregation":[88],"takes":[89],"granted":[91],"--":[92],"KV":[94,152],"format":[95],"both":[96],"ends":[97],"consume":[98],"natively,":[99],"shared":[102],"software":[103],"stack.":[104],"We":[105],"present":[106],"\\textbf{HMA-Serve},":[107],"MemHA-centric":[109],"disaggregated":[110],"system":[112],"pairing":[113],"efficiently.":[123],"HMA-Serve":[124,194],"achieves":[125],"this":[126],"through":[127],"(1)":[128],"phase-wise":[129],"quantization,":[130],"applying":[131],"vendor-native":[132],"low":[133],"precision":[134],"high-throughput":[136],"while":[138],"keeping":[139],"in":[141],"high-precision":[142],"BF16,":[143],"(2)":[144],"compute-transfer":[146],"pipeline":[147],"overlaps":[149],"layer's":[151],"cache":[153],"transfer":[154],"later-layer":[156],"reduce":[159,179],"time-to-first-token":[160],"(TTFT),":[161],"(3)":[163],"deferred":[164],"dequantization,":[165],"shipping":[166],"raw":[167],"quantized":[168],"bytes":[169],"reconstructing":[171],"lazily":[173],"GPU":[177],"network":[180],"usage.":[184],"Across":[185],"four":[186],"Qwen3":[187],"models":[188],"(4B--32B)":[189],"three":[191],"production":[192],"traces,":[193],"delivers":[195],"up":[196],"$3.2\\times$":[198],"higher":[199,207],"goodput":[200],"than":[201],"state-of-the-art":[202],"memory-homogeneous":[203],"methods":[204],"$4.8\\times$":[206],"goodput-per-dollar,":[208],"no":[210],"measurable":[211],"loss":[212],"generation-quality":[214],"benchmarks.":[215]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-01T00:00:00"}