{"id":"https://openalex.org/W4400448267","doi":"https://doi.org/10.1109/mm.2024.3420728","title":"A Latency Processing Unit: A Latency-Optimized and Highly Scalable Processor for Large Language Model Inference","display_name":"A Latency Processing Unit: A Latency-Optimized and Highly Scalable Processor for Large Language Model Inference","publication_year":2024,"publication_date":"2024-07-09","ids":{"openalex":"https://openalex.org/W4400448267","doi":"https://doi.org/10.1109/mm.2024.3420728"},"language":"en","primary_location":{"id":"doi:10.1109/mm.2024.3420728","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mm.2024.3420728","pdf_url":null,"source":{"id":"https://openalex.org/S59697426","display_name":"IEEE Micro","issn_l":"0272-1732","issn":["0272-1732","1937-4143"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Micro","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102300992","display_name":"Seungjae Moon","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Seungjae Moon","raw_affiliation_strings":["HyperAccel, Seoul, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100405467","display_name":"Jung\u2010Hoon Kim","orcid":"https://orcid.org/0000-0003-0471-6092"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jung-Hoon Kim","raw_affiliation_strings":["HyperAccel, Seoul, South Korea","Korea Advanced Institute of Science Technology, Daejeon, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]},{"raw_affiliation_string":"Korea Advanced Institute of Science Technology, Daejeon, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100636168","display_name":"Junsoo Kim","orcid":"https://orcid.org/0000-0002-4450-8741"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Junsoo Kim","raw_affiliation_strings":["HyperAccel, Seoul, South Korea","Korea Advanced Institute of Science Technology, Daejeon, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]},{"raw_affiliation_string":"Korea Advanced Institute of Science Technology, Daejeon, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075135160","display_name":"Seongmin Hong","orcid":"https://orcid.org/0000-0001-6904-8094"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Seongmin Hong","raw_affiliation_strings":["HyperAccel, Seoul, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092947137","display_name":"Junseo Cha","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Junseo Cha","raw_affiliation_strings":["HyperAccel, Seoul, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100343631","display_name":"Minsu Kim","orcid":"https://orcid.org/0000-0002-7179-6929"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Minsu Kim","raw_affiliation_strings":["Korea Advanced Institute of Science Technology, Daejeon, South Korea"],"affiliations":[{"raw_affiliation_string":"Korea Advanced Institute of Science Technology, Daejeon, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047063086","display_name":"Sukbin Lim","orcid":"https://orcid.org/0000-0001-5743-4852"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sukbin Lim","raw_affiliation_strings":["HyperAccel, Seoul, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113282211","display_name":"Gyubin Choi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gyubin Choi","raw_affiliation_strings":["HyperAccel, Seoul, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054918251","display_name":"Dongjin Seo","orcid":"https://orcid.org/0000-0002-1315-6456"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dongjin Seo","raw_affiliation_strings":["HyperAccel, Seoul, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100427730","display_name":"Jong-Ho Kim","orcid":"https://orcid.org/0000-0002-1309-0821"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jongho Kim","raw_affiliation_strings":["HyperAccel, Seoul, South Korea","Korea Advanced Institute of Science Technology, Daejeon, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]},{"raw_affiliation_string":"Korea Advanced Institute of Science Technology, Daejeon, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046228864","display_name":"Hunjong Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hunjong Lee","raw_affiliation_strings":["HyperAccel, Seoul, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054465836","display_name":"Hyun Jun Park","orcid":"https://orcid.org/0000-0003-0566-9574"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hyunjun Park","raw_affiliation_strings":["HyperAccel, Seoul, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046602398","display_name":"Ryeowook Ko","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ryeowook Ko","raw_affiliation_strings":["HyperAccel, Seoul, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111252036","display_name":"Soongyu Choi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Soongyu Choi","raw_affiliation_strings":["HyperAccel, Seoul, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037553165","display_name":"Jongse Park","orcid":"https://orcid.org/0000-0002-6629-449X"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jongse Park","raw_affiliation_strings":["Korea Advanced Institute of Science Technology, Daejeon, South Korea"],"affiliations":[{"raw_affiliation_string":"Korea Advanced Institute of Science Technology, Daejeon, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100750649","display_name":"Jinwon Lee","orcid":"https://orcid.org/0000-0003-4810-1014"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jinwon Lee","raw_affiliation_strings":["HyperAccel, Seoul, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100447377","display_name":"Joo-Young Kim","orcid":"https://orcid.org/0000-0003-1099-1496"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Joo-Young Kim","raw_affiliation_strings":["HyperAccel, Seoul, South Korea","Korea Advanced Institute of Science Technology, Daejeon, South Korea"],"affiliations":[{"raw_affiliation_string":"HyperAccel, Seoul, South Korea","institution_ids":[]},{"raw_affiliation_string":"Korea Advanced Institute of Science Technology, Daejeon, South Korea","institution_ids":["https://openalex.org/I157485424"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":17,"corresponding_author_ids":["https://openalex.org/A5102300992"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.7801,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.91450626,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"44","issue":"6","first_page":"17","last_page":"33"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.984000027179718,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.984000027179718,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.98089998960495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7951564788818359},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.7212679982185364},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.6944172382354736},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6738289594650269},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.46714991331100464},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.336401104927063},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.20917198061943054},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.12880301475524902}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7951564788818359},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7212679982185364},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.6944172382354736},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6738289594650269},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.46714991331100464},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.336401104927063},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.20917198061943054},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.12880301475524902},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/mm.2024.3420728","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mm.2024.3420728","pdf_url":null,"source":{"id":"https://openalex.org/S59697426","display_name":"IEEE Micro","issn_l":"0272-1732","issn":["0272-1732","1937-4143"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Micro","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W2034861439","https://openalex.org/W2979826702","https://openalex.org/W3006732000","https://openalex.org/W3159727696","https://openalex.org/W4281708879","https://openalex.org/W4308083513","https://openalex.org/W4385245566","https://openalex.org/W6739089268","https://openalex.org/W6771626834","https://openalex.org/W6810081322"],"related_works":["https://openalex.org/W2389214306","https://openalex.org/W4235240664","https://openalex.org/W2965083567","https://openalex.org/W1838576100","https://openalex.org/W2095886385","https://openalex.org/W2889616422","https://openalex.org/W2089704382","https://openalex.org/W2393741509","https://openalex.org/W3128807919","https://openalex.org/W3176411177"],"abstract_inverted_index":{"The":[0,50,67,97,122],"explosive":[1],"arrival":[2],"of":[3,10,17,19,26,47,133,139],"OpenAI\u2019s":[4],"ChatGPT":[5],"has":[6,129],"fueled":[7],"the":[8,24,45,86,106],"globalization":[9],"large":[11],"language":[12],"models":[13],"(LLMs),":[14],"which":[15,112],"consist":[16],"billions":[18],"pretrained":[20],"parameters":[21],"that":[22,76],"embody":[23],"aspects":[25],"syntax":[27],"and":[28,39,56,65,102,108,115,136,146,153],"semantics.":[29],"HyperAccel":[30],"introduces":[31],"a":[32,37,120,130],"latency":[33,80],"processing":[34],"unit":[35],"(LPU),":[36],"latency-optimized":[38],"highly":[40],"scalable":[41],"processor":[42],"architecture":[43],"for":[44,105],"acceleration":[46],"LLM":[48,95],"inference.":[49],"LPU":[51,68,87,98],"perfectly":[52],"balances":[53],"memory":[54],"bandwidth":[55],"compute":[57],"logic":[58],"with":[59,71],"streamlined":[60],"dataflow":[61],"to":[62,93],"maximize":[63],"performance":[64],"efficiency.":[66],"is":[69,113],"equipped":[70],"an":[72,89],"expandable":[73],"synchronization":[74,79],"link":[75],"hides":[77],"data":[78],"among":[81],"multiple":[82],"LPUs.":[83],"HyperDex":[84],"complements":[85],"as":[88],"intuitive":[90],"software":[91],"framework":[92],"run":[94],"applications.":[96],"achieves":[99],"1.25":[100],"ms/token":[101,104],"20.9":[103],"1.3B":[107],"66B":[109],"models,":[110],"respectively,":[111,118],"2.09\u00d7":[114],"1.37\u00d7":[116],"faster,":[117],"than":[119],"GPU.":[121],"LPU,":[123],"synthesized":[124],"using":[125],"Samsung\u2019s":[126],"4-nm":[127],"process,":[128],"total":[131],"area":[132],"0.824":[134],"mm2":[135],"power":[137],"consumption":[138],"284.31":[140],"mW.":[141],"LPU-based":[142],"servers":[143],"achieve":[144],"1.33\u00d7":[145],"1.32\u00d7":[147],"energy":[148],"efficiency":[149],"over":[150],"Nvidia\u2019s":[151],"H100":[152],"L4":[154],"servers,":[155],"respectively.":[156]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":5}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2024-07-10T00:00:00"}