{"id":"https://openalex.org/W7162448395","doi":"https://doi.org/10.48550/arxiv.2605.25655","title":"Bandwidth-Aware LLM Inference on Heterogeneous Many-Core Supercomputers","display_name":"Bandwidth-Aware LLM Inference on Heterogeneous Many-Core Supercomputers","publication_year":2026,"publication_date":"2026-05-25","ids":{"openalex":"https://openalex.org/W7162448395","doi":"https://doi.org/10.48550/arxiv.2605.25655"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.25655","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.25655","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.25655","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137004422","display_name":"Yao Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Yao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137043879","display_name":"Zhongzhi Luan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luan, Zhongzhi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137022696","display_name":"GEN LI","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Gen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137049870","display_name":"Jiaxing Qi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi, Jiaxing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137084662","display_name":"Shiqing Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Shiqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137034716","display_name":"Bin Han","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Bin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137079556","display_name":"Shizhe Shang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shang, Shizhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137035905","display_name":"Hailong Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Hailong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137053390","display_name":"Depei Qian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qian, Depei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.22280000150203705,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.22280000150203705,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.11940000206232071,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.10769999772310257,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/memory-bandwidth","display_name":"Memory bandwidth","score":0.6360999941825867},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5788000226020813},{"id":"https://openalex.org/keywords/supercomputer","display_name":"Supercomputer","score":0.529699981212616},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.5073999762535095},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.4763000011444092},{"id":"https://openalex.org/keywords/memory-hierarchy","display_name":"Memory hierarchy","score":0.42570000886917114},{"id":"https://openalex.org/keywords/simd","display_name":"SIMD","score":0.41909998655319214},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.38429999351501465},{"id":"https://openalex.org/keywords/xeon","display_name":"Xeon","score":0.38029998540878296}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8790000081062317},{"id":"https://openalex.org/C188045654","wikidata":"https://www.wikidata.org/wiki/Q17148339","display_name":"Memory bandwidth","level":2,"score":0.6360999941825867},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.608299970626831},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5788000226020813},{"id":"https://openalex.org/C83283714","wikidata":"https://www.wikidata.org/wiki/Q121117","display_name":"Supercomputer","level":2,"score":0.529699981212616},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.5073999762535095},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.4763000011444092},{"id":"https://openalex.org/C2778100165","wikidata":"https://www.wikidata.org/wiki/Q1589327","display_name":"Memory hierarchy","level":3,"score":0.42570000886917114},{"id":"https://openalex.org/C150552126","wikidata":"https://www.wikidata.org/wiki/Q339387","display_name":"SIMD","level":2,"score":0.41909998655319214},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.38429999351501465},{"id":"https://openalex.org/C145108525","wikidata":"https://www.wikidata.org/wiki/Q656154","display_name":"Xeon","level":2,"score":0.38029998540878296},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3668000102043152},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.3603000044822693},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.35600000619888306},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.3443000018596649},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.33970001339912415},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.33570000529289246},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.328900009393692},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.32010000944137573},{"id":"https://openalex.org/C128099668","wikidata":"https://www.wikidata.org/wiki/Q573952","display_name":"Lazy evaluation","level":3,"score":0.3197999894618988},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.3147999942302704},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.3028999865055084},{"id":"https://openalex.org/C12186640","wikidata":"https://www.wikidata.org/wiki/Q6815743","display_name":"Memory model","level":3,"score":0.29409998655319214},{"id":"https://openalex.org/C2776834041","wikidata":"https://www.wikidata.org/wiki/Q25346349","display_name":"Execution model","level":2,"score":0.2896000146865845},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.28450000286102295},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.2806999981403351},{"id":"https://openalex.org/C96324660","wikidata":"https://www.wikidata.org/wiki/Q205446","display_name":"Dataflow","level":2,"score":0.27900001406669617},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.2766999900341034},{"id":"https://openalex.org/C2778915421","wikidata":"https://www.wikidata.org/wiki/Q3643177","display_name":"Performance improvement","level":2,"score":0.27320000529289246},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.2551000118255615},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.2533000111579895},{"id":"https://openalex.org/C27602214","wikidata":"https://www.wikidata.org/wiki/Q1868547","display_name":"Locality of reference","level":3,"score":0.2524000108242035},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.25200000405311584}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.25655","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.25655","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.25655","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.25655","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"model":[2,162,172],"(LLM)":[3],"inference":[4,53,64,247],"is":[5],"limited":[6,35],"by":[7,173,186],"high":[8],"computational":[9],"cost":[10],"and":[11,38,76,117,130,136,145,156,185,198,234,244],"memory":[12,40],"bandwidth":[13,37],"demands,":[14],"making":[15,45],"deployment":[16],"on":[17,154,159,169,181,212,248],"heterogeneous":[18,249],"many-core":[19,250],"processors":[20],"challenging.":[21],"Taking":[22],"the":[23,28,91,106,160,170,193,213,224],"MT-3000":[24],"processor":[25],"used":[26],"in":[27],"Tianhe":[29],"supercomputer":[30],"as":[31],"an":[32],"example,":[33],"its":[34],"main-memory":[36],"distributed":[39],"hierarchy":[41],"exemplify":[42],"these":[43],"bottlenecks,":[44],"it":[46],"difficult":[47],"to":[48,102,176,189,221],"directly":[49],"migrate":[50],"existing":[51],"GPU-based":[52,218],"frameworks.":[54],"To":[55],"address":[56],"this":[57],"problem,":[58],"we":[59],"propose":[60],"THInfer,":[61],"a":[62,86,112,124,132,238],"hardware-aware":[63],"framework":[65],"that":[66,99,165],"maximizes":[67],"data":[68],"locality":[69],"under":[70,223],"bandwidth-constrained":[71],"conditions":[72],"through":[73,150],"hardware-software":[74],"co-design":[75],"parallel":[77],"strategy":[78],"optimization.":[79],"THInfer":[80,166,208,228],"incorporates":[81],"three":[82],"key":[83],"techniques:":[84],"(1)":[85],"high-performance":[87],"operator":[88],"library":[89],"for":[90,242],"VLIW":[92],"SIMD":[93],"architecture,":[94],"providing":[95,237],"hand-optimized":[96],"FP16":[97],"kernels":[98],"achieve":[100],"up":[101],"70":[103],"percent":[104,175,178,188,191],"of":[105],"peak":[107],"performance":[108,211],"per":[109],"cluster;":[110],"(2)":[111],"density-driven":[113],"computation":[114],"graph":[115],"fusion":[116,128],"unified":[118],"kernel":[119],"scheduling":[120],"mechanism,":[121],"combined":[122],"with":[123],"staged":[125],"pipelined":[126],"attention":[127],"method;":[129],"(3)":[131],"Prefill-Buffer-Decode":[133],"(P-B-D)":[134],"pipeline":[135],"bounded":[137],"buffer":[138],"management":[139],"strategy,":[140],"which":[141],"supports":[142],"hybrid":[143],"parallelism":[144],"enables":[146],"efficient":[147,243],"multi-cluster":[148],"collaboration":[149],"two-level":[151],"communication":[152],"based":[153],"MPI":[155],"hthreads.":[157],"Experiments":[158],"Llama":[161],"series":[163],"show":[164],"improves":[167,235],"throughput":[168],"7B":[171],"62":[174],"73":[177],"over":[179,192],"DeepSpeed":[180],"two":[182],"V100S":[183],"GPUs":[184],"67":[187],"84":[190],"A800":[194],"GPU.":[195],"The":[196],"13B":[197],"30B":[199],"models":[200],"also":[201],"demonstrate":[202],"comparable":[203],"or":[204],"better":[205],"performance.":[206],"Moreover,":[207],"maintains":[209],"stable":[210],"70B":[214],"model,":[215],"whereas":[216],"typical":[217],"frameworks":[219],"fail":[220],"run":[222],"same":[225],"setting.":[226],"Overall,":[227],"significantly":[229],"enhances":[230],"throughput,":[231],"reduces":[232],"latency,":[233],"scalability,":[236],"feasible":[239],"system":[240],"solution":[241],"scalable":[245],"LLM":[246],"architectures.":[251]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-27T00:00:00"}