{"id":"https://openalex.org/W7153979821","doi":"https://doi.org/10.1145/3807449","title":"Optimizing Attention for Large Language Model Inference on the MT-3000 Many-Core Processor","display_name":"Optimizing Attention for Large Language Model Inference on the MT-3000 Many-Core Processor","publication_year":2026,"publication_date":"2026-04-13","ids":{"openalex":"https://openalex.org/W7153979821","doi":"https://doi.org/10.1145/3807449"},"language":"en","primary_location":{"id":"doi:10.1145/3807449","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3807449","pdf_url":null,"source":{"id":"https://openalex.org/S26056741","display_name":"ACM Transactions on Architecture and Code Optimization","issn_l":"1544-3566","issn":["1544-3566","1544-3973"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Architecture and Code Optimization","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1145/3807449","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065038356","display_name":"Xinxin Qi","orcid":"https://orcid.org/0000-0001-8316-2934"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinxin Qi","raw_affiliation_strings":["National University of Defense Technology"],"raw_orcid":"https://orcid.org/0000-0001-8316-2934","affiliations":[{"raw_affiliation_string":"National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083171604","display_name":"Jianbin Fang","orcid":"https://orcid.org/0000-0003-3542-4869"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianbin Fang","raw_affiliation_strings":["National University of Defense Technology"],"raw_orcid":"https://orcid.org/0000-0003-3542-4869","affiliations":[{"raw_affiliation_string":"National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092019932","display_name":"Peng Zhang","orcid":"https://orcid.org/0000-0001-8364-9793"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peng Zhang","raw_affiliation_strings":["National University of Defense Technology"],"raw_orcid":"https://orcid.org/0000-0001-8364-9793","affiliations":[{"raw_affiliation_string":"National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008369819","display_name":"Yonggang Che","orcid":"https://orcid.org/0000-0001-6906-4940"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yonggang Che","raw_affiliation_strings":["National University of Defense Technology"],"raw_orcid":"https://orcid.org/0000-0001-6906-4940","affiliations":[{"raw_affiliation_string":"National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I170215575"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.36472984,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"23","issue":"2","first_page":"1","last_page":"27"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.23929999768733978,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.23929999768733978,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.21879999339580536,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.14550000429153442,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.6571999788284302},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.5156999826431274},{"id":"https://openalex.org/keywords/modular-design","display_name":"Modular design","score":0.5097000002861023},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5008000135421753},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.47519999742507935},{"id":"https://openalex.org/keywords/reuse","display_name":"Reuse","score":0.4672999978065491},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.46389999985694885},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.43470001220703125},{"id":"https://openalex.org/keywords/reduction","display_name":"Reduction (mathematics)","score":0.4083000123500824},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.3968999981880188}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.900600016117096},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6571999788284302},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.5156999826431274},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.5097000002861023},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5008000135421753},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.47519999742507935},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.4672999978065491},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.46389999985694885},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.43470001220703125},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.4083000123500824},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.3968999981880188},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.38440001010894775},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.37130001187324524},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3634999990463257},{"id":"https://openalex.org/C17020691","wikidata":"https://www.wikidata.org/wiki/Q139677","display_name":"Operator (biology)","level":5,"score":0.34369999170303345},{"id":"https://openalex.org/C2776834041","wikidata":"https://www.wikidata.org/wiki/Q25346349","display_name":"Execution model","level":2,"score":0.34279999136924744},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3264000117778778},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.3237000107765198},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.32199999690055847},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.32089999318122864},{"id":"https://openalex.org/C34736171","wikidata":"https://www.wikidata.org/wiki/Q918333","display_name":"Preprocessor","level":2,"score":0.3098999857902527},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30979999899864197},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.30570000410079956},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.304500013589859},{"id":"https://openalex.org/C128099668","wikidata":"https://www.wikidata.org/wiki/Q573952","display_name":"Lazy evaluation","level":3,"score":0.30320000648498535},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2849999964237213},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2775000035762787},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.2750000059604645},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.27239999175071716},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.2687000036239624},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.2605000138282776},{"id":"https://openalex.org/C158600405","wikidata":"https://www.wikidata.org/wiki/Q5054566","display_name":"Causal inference","level":2,"score":0.25870001316070557},{"id":"https://openalex.org/C10551718","wikidata":"https://www.wikidata.org/wiki/Q5227332","display_name":"Data pre-processing","level":2,"score":0.2556000053882599},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.25119999051094055}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3807449","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3807449","pdf_url":null,"source":{"id":"https://openalex.org/S26056741","display_name":"ACM Transactions on Architecture and Code Optimization","issn_l":"1544-3566","issn":["1544-3566","1544-3973"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Architecture and Code Optimization","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3807449","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3807449","pdf_url":null,"source":{"id":"https://openalex.org/S26056741","display_name":"ACM Transactions on Architecture and Code Optimization","issn_l":"1544-3566","issn":["1544-3566","1544-3973"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Architecture and Code Optimization","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5120968873","display_name":null,"funder_award_id":"62421002 and 62302505","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W2963341956","https://openalex.org/W2973166032","https://openalex.org/W4281390154","https://openalex.org/W4282962596","https://openalex.org/W4320067900","https://openalex.org/W4321636575","https://openalex.org/W4378227035","https://openalex.org/W4378697133","https://openalex.org/W4387321091","https://openalex.org/W4391923867","https://openalex.org/W4392427708","https://openalex.org/W4393186054","https://openalex.org/W4400411561","https://openalex.org/W4415797154"],"related_works":[],"abstract_inverted_index":{"Transformer-based":[0],"large":[1,134],"language":[2],"models":[3],"(LLM)":[4],"are":[5,35],"increasingly":[6],"deployed":[7,52],"in":[8,53,63],"high-performance":[9,65,105],"computing":[10],"environments,":[11],"where":[12],"the":[13,54,110,125,187,191,197],"attention":[14,25,106,119],"mechanism":[15],"often":[16],"becomes":[17],"a":[18,49,104,116,139,166],"key":[19],"bottleneck":[20],"during":[21],"inference.":[22,207],"Although":[23],"state-of-the-art":[24],"algorithms":[26],"(e.g.,":[27],"FlashAttention)":[28],"achieve":[29],"high":[30],"efficiency":[31],"on":[32,47],"GPUs,":[33],"they":[34],"ill-suited":[36],"to":[37,88,90,121,152,160,184],"emerging":[38],"heterogeneous":[39],"many-core":[40,112],"processors.":[41],"In":[42],"this":[43],"work,":[44],"we":[45,100],"focus":[46],"MT-3000,":[48],"representative":[50],"architecture":[51],"new-generation":[55],"Tianhe":[56],"supercomputer,":[57],"and":[58,82,93,149,156,194,203],"identify":[59],"three":[60],"principal":[61],"challenges":[62],"realizing":[64],"attention:":[66],"complex":[67],"multi-tier":[68],"memory":[69],"requiring":[70],"manual":[71],"data":[72,144,154],"movement,":[73],"excessive":[74],"reduction":[75,123],"overhead":[76],"caused":[77],"by":[78,172],"sub-tile":[79],"softmax":[80],"operations,":[81],"static":[83],"execution":[84,157],"pipelines":[85],"that":[86,180],"fail":[87],"adapt":[89],"inference":[91],"phases":[92],"sequence":[94],"lengths.":[95],"To":[96],"overcome":[97],"these":[98],"challenges,":[99],"propose":[101],"DeferAttention":[102,114,137,164,181],",":[103],"implementation":[107],"designed":[108],"for":[109],"MT-3000":[111],"processor.":[113],"introduces":[115],"novel":[117],"deferred-reduction":[118],"strategy":[120,170],"decouple":[122],"from":[124],"fused":[126],"compute":[127],"pipeline,":[128],"enabling":[129],"more":[130],"efficient":[131],"aggregation":[132],"over":[133],"tiles.":[135],"Moreover,":[136],"adopts":[138],"memory-centric":[140],"operator":[141,198],"design,":[142],"including":[143],"tiling,":[145],"multi-level":[146],"software":[147],"pipelining,":[148],"modular":[150],"micro-kernels,":[151],"maximize":[153],"reuse":[155],"throughput.":[158],"Finally,":[159],"support":[161],"runtime-adaptive":[162],"execution,":[163],"integrates":[165],"lightweight":[167],"kernel":[168],"selection":[169],"guided":[171],"an":[173],"analytical":[174],"cost":[175],"model.":[176],"Experimental":[177],"results":[178],"show":[179],"achieves":[182],"up":[183],"98%":[185],"of":[186],"theoretical":[188],"peak":[189],"at":[190,196],"micro-kernel":[192],"level":[193],"85%":[195],"level,":[199],"outperforming":[200],"baseline":[201],"implementations":[202],"significantly":[204],"accelerating":[205],"end-to-end":[206]},"counts_by_year":[],"updated_date":"2026-06-28T06:15:30.350997","created_date":"2026-04-14T00:00:00"}