{"id":"https://openalex.org/W4378697133","doi":"https://doi.org/10.1109/tpds.2023.3280805","title":"Full-Stack Optimizing Transformer Inference on ARM Many-Core CPU","display_name":"Full-Stack Optimizing Transformer Inference on ARM Many-Core CPU","publication_year":2023,"publication_date":"2023-05-29","ids":{"openalex":"https://openalex.org/W4378697133","doi":"https://doi.org/10.1109/tpds.2023.3280805"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2023.3280805","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2023.3280805","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048148542","display_name":"Jiazhi Jiang","orcid":"https://orcid.org/0000-0002-1417-3012"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]},{"id":"https://openalex.org/I4210112812","display_name":"National Supercomputing Center in Shenzhen","ror":"https://ror.org/02291hh73","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210112812"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiazhi Jiang","raw_affiliation_strings":["School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","National Supercomputer Center in Guangzhou, Guangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-1417-3012","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]},{"raw_affiliation_string":"National Supercomputer Center in Guangzhou, Guangzhou, China","institution_ids":["https://openalex.org/I4210112812"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024085713","display_name":"Jiangsu Du","orcid":"https://orcid.org/0000-0003-4707-9492"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]},{"id":"https://openalex.org/I4210112812","display_name":"National Supercomputing Center in Shenzhen","ror":"https://ror.org/02291hh73","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210112812"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiangsu Du","raw_affiliation_strings":["School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","National Supercomputer Center in Guangzhou, Guangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-4707-9492","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]},{"raw_affiliation_string":"National Supercomputer Center in Guangzhou, Guangzhou, China","institution_ids":["https://openalex.org/I4210112812"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041534890","display_name":"Dan Huang","orcid":"https://orcid.org/0000-0001-5582-1031"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]},{"id":"https://openalex.org/I4210112812","display_name":"National Supercomputing Center in Shenzhen","ror":"https://ror.org/02291hh73","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210112812"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dan Huang","raw_affiliation_strings":["School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","National Supercomputer Center in Guangzhou, Guangzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-5582-1031","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]},{"raw_affiliation_string":"National Supercomputer Center in Guangzhou, Guangzhou, China","institution_ids":["https://openalex.org/I4210112812"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101483479","display_name":"Zhiguang Chen","orcid":"https://orcid.org/0000-0002-9318-5715"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]},{"id":"https://openalex.org/I4210112812","display_name":"National Supercomputing Center in Shenzhen","ror":"https://ror.org/02291hh73","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210112812"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiguang Chen","raw_affiliation_strings":["School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","National Supercomputer Center in Guangzhou, Guangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-9318-5715","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]},{"raw_affiliation_string":"National Supercomputer Center in Guangzhou, Guangzhou, China","institution_ids":["https://openalex.org/I4210112812"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101633465","display_name":"Yutong Lu","orcid":"https://orcid.org/0000-0001-5315-3375"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]},{"id":"https://openalex.org/I4210112812","display_name":"National Supercomputing Center in Shenzhen","ror":"https://ror.org/02291hh73","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210112812"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yutong Lu","raw_affiliation_strings":["School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","National Supercomputer Center in Guangzhou, Guangzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]},{"raw_affiliation_string":"National Supercomputer Center in Guangzhou, Guangzhou, China","institution_ids":["https://openalex.org/I4210112812"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5000688713","display_name":"Xiangke Liao","orcid":"https://orcid.org/0000-0002-6125-3330"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]},{"id":"https://openalex.org/I4210112812","display_name":"National Supercomputing Center in Shenzhen","ror":"https://ror.org/02291hh73","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210112812"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiangke Liao","raw_affiliation_strings":["School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","National Supercomputer Center in Guangzhou, Guangzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]},{"raw_affiliation_string":"National Supercomputer Center in Guangzhou, Guangzhou, China","institution_ids":["https://openalex.org/I4210112812"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.976,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.94566624,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"34","issue":"7","first_page":"2221","last_page":"2235"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8932548761367798},{"id":"https://openalex.org/keywords/multi-core-processor","display_name":"Multi-core processor","score":0.566389799118042},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.5509228706359863},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5418317317962646},{"id":"https://openalex.org/keywords/thread","display_name":"Thread (computing)","score":0.5160481333732605},{"id":"https://openalex.org/keywords/central-processing-unit","display_name":"Central processing unit","score":0.48069077730178833},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.46016499400138855},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4465046226978302},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.4456843137741089},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.4139534533023834},{"id":"https://openalex.org/keywords/instruction-set","display_name":"Instruction set","score":0.411823570728302},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.24627956748008728},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.1517011821269989},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.10800594091415405}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8932548761367798},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.566389799118042},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.5509228706359863},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5418317317962646},{"id":"https://openalex.org/C138101251","wikidata":"https://www.wikidata.org/wiki/Q213092","display_name":"Thread (computing)","level":2,"score":0.5160481333732605},{"id":"https://openalex.org/C49154492","wikidata":"https://www.wikidata.org/wiki/Q5300","display_name":"Central processing unit","level":2,"score":0.48069077730178833},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.46016499400138855},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4465046226978302},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.4456843137741089},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4139534533023834},{"id":"https://openalex.org/C202491316","wikidata":"https://www.wikidata.org/wiki/Q272683","display_name":"Instruction set","level":2,"score":0.411823570728302},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.24627956748008728},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.1517011821269989},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.10800594091415405},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2023.3280805","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2023.3280805","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W2084379367","https://openalex.org/W2186615578","https://openalex.org/W2475126267","https://openalex.org/W2804032941","https://openalex.org/W2804500013","https://openalex.org/W2888498443","https://openalex.org/W2896457183","https://openalex.org/W2924040443","https://openalex.org/W2965373594","https://openalex.org/W2978017171","https://openalex.org/W3004659153","https://openalex.org/W3041159379","https://openalex.org/W3114874950","https://openalex.org/W3130716829","https://openalex.org/W3130954105","https://openalex.org/W3132840842","https://openalex.org/W3177265267","https://openalex.org/W3195800062","https://openalex.org/W3202294838","https://openalex.org/W3210601829","https://openalex.org/W3211060193","https://openalex.org/W4281758439","https://openalex.org/W4295312788","https://openalex.org/W4320067900","https://openalex.org/W6686509673","https://openalex.org/W6751349269","https://openalex.org/W6752057402","https://openalex.org/W6755207826","https://openalex.org/W6766673545","https://openalex.org/W6766978945","https://openalex.org/W6768807518","https://openalex.org/W6768851824","https://openalex.org/W6790838377","https://openalex.org/W6803617553","https://openalex.org/W6838322825"],"related_works":["https://openalex.org/W17155033","https://openalex.org/W3207760230","https://openalex.org/W1496222301","https://openalex.org/W2473478803","https://openalex.org/W2003735","https://openalex.org/W2168921806","https://openalex.org/W2729363167","https://openalex.org/W2060611139","https://openalex.org/W2499494780","https://openalex.org/W2355246969"],"abstract_inverted_index":{"The":[0,152],"past":[1],"several":[2,72],"years":[3],"have":[4],"witnessed":[5],"tremendous":[6],"success":[7],"of":[8,75,90,135,165],"transformer":[9,65,166],"models":[10],"in":[11],"natural":[12],"language":[13],"processing":[14],"(NLP),":[15],"and":[16,30,57,95,144,178,184],"their":[17],"current":[18],"landscape":[19],"is":[20,61,155,176,189],"increasingly":[21],"diverse.":[22],"Although":[23],"GPU":[24],"gradually":[25],"becomes":[26],"the":[27,123,136,163],"dominating":[28],"workhorse":[29],"de":[31],"facto":[32],"standard":[33],"for":[34,98,109,148,157,191],"deep":[35],"learning,":[36],"there":[37],"are":[38,107],"still":[39],"many":[40],"scenarios":[41],"where":[42],"using":[43],"CPU":[44,81,128],"remains":[45],"a":[46,145,172,185],"prevalent":[47],"choice.Recently,":[48],"ARM":[49,223],"many-core":[50,80,224],"processor":[51],"starts":[52],"emigrating":[53],"to":[54,63,115,126,161,180,207,218],"cloud":[55],"computing":[56],"high-performance":[58],"computing,":[59],"which":[60],"promising":[62],"deploy":[64],"inference.":[66],"In":[67],"this":[68],"paper,":[69],"we":[70,139],"identify":[71],"performance":[73],"bottlenecks":[74],"existing":[76,203],"inference":[77,137],"runtime":[78],"on":[79,221],"including":[82],"low-core":[83],"usage,":[84],"isolated":[85],"thread":[86,142],"configuration,":[87],"inappropriate":[88],"implementation":[89,154,200],"general":[91],"matrix":[92],"multiply":[93],"(GEMM),":[94],"redundant":[96,170],"computations":[97],"variable-length":[99,213],"inputs.":[100,211],"To":[101,131,168],"tackle":[102],"these":[103,110],"problems,":[104],"full-stack":[105],"optimizations":[106],"conducted":[108],"challenges":[111],"from":[112],"service":[113,124],"level":[114,125],"operator":[116],"level.":[117],"We":[118],"explore":[119],"multi-instance":[120],"parallelization":[121],"at":[122],"improve":[127,132],"core":[129],"usage.":[130],"parallel":[133,150],"efficiency":[134],"runtime,":[138],"design":[140],"NUMA-aware":[141],"scheduling":[143],"look-up":[146],"table":[147],"optimal":[149],"configurations.":[151],"GEMM":[153],"tailored":[156],"some":[158],"critical":[159],"modules":[160],"exploit":[162],"characteristics":[164],"workload.":[167],"eliminate":[169],"computations,":[171],"novel":[173],"storage":[174],"format":[175],"designed":[177],"implemented":[179],"pack":[181],"sparse":[182],"data":[183],"load":[186],"balancing":[187],"strategy":[188],"proposed":[190],"tasks":[192],"with":[193,209],"different":[194,222],"sparsity.":[195],"Experiments":[196],"show":[197],"that":[198],"our":[199],"can":[201],"outperform":[202],"solutions":[204],"by":[205],"1.1x":[206],"6x":[208],"fixed-length":[210],"For":[212],"inputs,":[214],"it":[215],"achieves":[216],"1.9x":[217],"8x":[219],"speedups":[220],"processors.":[225]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}