{"id":"https://openalex.org/W7166693320","doi":"https://doi.org/10.48550/arxiv.2606.29708","title":"Demystifying the Design Space and Best Practices for Heterogeneous LLM Inference and Serving","display_name":"Demystifying the Design Space and Best Practices for Heterogeneous LLM Inference and Serving","publication_year":2026,"publication_date":"2026-06-29","ids":{"openalex":"https://openalex.org/W7166693320","doi":"https://doi.org/10.48550/arxiv.2606.29708"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.29708","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29708","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.29708","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100641428","display_name":"Zhixin Wang","orcid":"https://orcid.org/0000-0001-7255-6049"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zhixin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074836725","display_name":"Zhengbo Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zhengbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139674214","display_name":"Fangcheng Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Fangcheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070644005","display_name":"Yinhui Lu","orcid":"https://orcid.org/0000-0001-9034-0823"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Yinhui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139635026","display_name":"Jinlong Hou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hou, Jinlong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139648059","display_name":"Yijie Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yijie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024169079","display_name":"Xiaowei Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Xiaowei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139653906","display_name":"He Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, He","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139712293","display_name":"Xiangbin Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xiangbin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139645245","display_name":"Jun Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139713411","display_name":"Ruya Gu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gu, Ruya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138730156","display_name":"Dian Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Dian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139698968","display_name":"Zhou Tan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tan, Zhou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139679460","display_name":"Yuan Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Yuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139657086","display_name":"Hongzhou Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Hongzhou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074455169","display_name":"XJ Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Xiangjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056795737","display_name":"P Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Ping","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139696088","display_name":"Xiaohe Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Xiaohe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10904","display_name":"Embedded Systems Design Techniques","score":0.43389999866485596,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10904","display_name":"Embedded Systems Design Techniques","score":0.43389999866485596,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.19130000472068787,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10142","display_name":"Formal Methods in Verification","score":0.04899999871850014,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.7141000032424927},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.7055000066757202},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.5099999904632568},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.45829999446868896},{"id":"https://openalex.org/keywords/byte","display_name":"Byte","score":0.4562999904155731},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.4025000035762787},{"id":"https://openalex.org/keywords/boundary","display_name":"Boundary (topology)","score":0.3411000072956085}],"concepts":[{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.7141000032424927},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7055000066757202},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6991999745368958},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.5099999904632568},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.45829999446868896},{"id":"https://openalex.org/C43364308","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Byte","level":2,"score":0.4562999904155731},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.4025000035762787},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3540000021457672},{"id":"https://openalex.org/C62354387","wikidata":"https://www.wikidata.org/wiki/Q875399","display_name":"Boundary (topology)","level":2,"score":0.3411000072956085},{"id":"https://openalex.org/C2776221188","wikidata":"https://www.wikidata.org/wiki/Q21072556","display_name":"Design space exploration","level":2,"score":0.3160000145435333},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.3138999938964844},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.31380000710487366},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.31299999356269836},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.3109999895095825},{"id":"https://openalex.org/C2778648169","wikidata":"https://www.wikidata.org/wiki/Q967768","display_name":"Compatibility (geochemistry)","level":2,"score":0.3057999908924103},{"id":"https://openalex.org/C20574231","wikidata":"https://www.wikidata.org/wiki/Q844605","display_name":"Backward compatibility","level":2,"score":0.2865000069141388},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.2831000089645386},{"id":"https://openalex.org/C20136886","wikidata":"https://www.wikidata.org/wiki/Q749647","display_name":"Interoperability","level":2,"score":0.27000001072883606}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.29708","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29708","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.29708","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29708","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.4217755198478699,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Heterogeneous":[0],"prefill-decode":[1],"(PD)":[2],"inference":[3,99],"is":[4,37,39],"now":[5],"in":[6,24,205],"production:":[7],"prefill":[8,183],"on":[9,15,33,144],"cost-efficient":[10],"or":[11],"supply-available":[12],"accelerators,":[13],"decode":[14,185],"bandwidth-strong":[16],"ones,":[17],"and":[18,53,71,74,114,168,179,184,186,196,209],"KV":[19,72,112,115,150,172],"state":[20],"crossing":[21],"mixed":[22,25],"interconnects":[23],"numerical":[26],"formats.":[27],"Each":[28],"deployment":[29,207],"makes":[30],"these":[31,92],"decisions":[32,44],"its":[34],"own.":[35],"What":[36],"missing":[38],"the":[40,50,75,137,148,213],"picture":[41],"across":[42],"configurations-which":[43],"must":[45],"be":[46,56],"made":[47,57],"jointly":[48],"at":[49],"PD":[51,98],"boundary,":[52],"which":[54],"can":[55],"independently.":[58],"We":[59,83],"propose":[60],"a":[61,87,132,176],"design":[62,67,202],"space":[63],"organized":[64],"along":[65],"four":[66],"axes-accelerator,":[68],"precision,":[69],"interconnect,":[70],"residency":[73],"workload":[76],"regime":[77],"(stage":[78],"pressure)":[79],"they":[80],"respond":[81],"to.":[82],"show":[84],"that":[85],"only":[86],"subset":[88],"of":[89,147,212],"interactions":[90,103,192],"among":[91],"factors":[93],"become":[94],"binding":[95],"constraints":[96],"once":[97],"becomes":[100],"heterogeneous.":[101],"These":[102],"surface":[104],"through":[105],"three":[106],"recurring":[107],"boundary":[108,164],"decisions:":[109],"compute":[110],"placement,":[111],"representation,":[113],"ownership.":[116,189],"The":[117,171],"resulting":[118],"analysis":[119],"yields":[120],"concrete":[121],"guidance.":[122],"Precision":[123],"policy":[124],"belongs":[125],"to":[126,131],"runtime":[127],"roles":[128],"rather":[129,155],"than":[130,156],"single":[133],"system-wide":[134],"setting,":[135],"because":[136],"same":[138],"low-bit":[139],"format":[140],"relieves":[141],"different":[142],"bottlenecks":[143],"each":[145],"side":[146],"boundary.":[149],"transfer":[151],"engines":[152],"move":[153],"bytes":[154],"tensor":[157],"semantics,":[158],"making":[159],"representation":[160],"compatibility":[161],"an":[162],"explicit":[163,188],"concern":[165],"whenever":[166],"producer":[167],"consumer":[169],"differ.":[170],"handoff":[173],"also":[174],"carries":[175],"lifecycle-reservation,":[177],"release,":[178],"failure":[180],"recovery-that":[181],"spans":[182],"requires":[187],"Two":[190],"further":[191],"remain":[193],"open.":[194],"Cross-vendor":[195],"interconnect-related":[197],"claims":[198],"are":[199],"stated":[200],"as":[201],"guidance":[203],"grounded":[204],"industrial":[206],"observations":[208],"source-code":[210],"inspection":[211],"runtimes":[214],"involved.":[215]},"counts_by_year":[],"updated_date":"2026-07-01T06:29:00.853634","created_date":"2026-07-01T00:00:00"}