{"id":"https://openalex.org/W4389888285","doi":"https://doi.org/10.1109/a-sscc58667.2023.10347930","title":"CIMFormer: A 38.9TOPS/W-8b Systolic CIM-Array Based Transformer Processor with Token-Slimmed Attention Reformulating and Principal Possibility Gathering","display_name":"CIMFormer: A 38.9TOPS/W-8b Systolic CIM-Array Based Transformer Processor with Token-Slimmed Attention Reformulating and Principal Possibility Gathering","publication_year":2023,"publication_date":"2023-11-05","ids":{"openalex":"https://openalex.org/W4389888285","doi":"https://doi.org/10.1109/a-sscc58667.2023.10347930"},"language":"en","primary_location":{"id":"doi:10.1109/a-sscc58667.2023.10347930","is_oa":false,"landing_page_url":"https://doi.org/10.1109/a-sscc58667.2023.10347930","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Asian Solid-State Circuits Conference (A-SSCC)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085105148","display_name":"Ruiqi Guo","orcid":"https://orcid.org/0000-0003-4729-7385"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruiqi Guo","raw_affiliation_strings":["Tsinghua University,BNRist,Beijing,China","BNRist, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,BNRist,Beijing,China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100322708","display_name":"Yang Wang","orcid":"https://orcid.org/0000-0002-5117-7055"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yang Wang","raw_affiliation_strings":["Tsinghua University,BNRist,Beijing,China","BNRist, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,BNRist,Beijing,China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101434816","display_name":"Xiaofeng Chen","orcid":"https://orcid.org/0000-0001-8572-7310"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaofeng Chen","raw_affiliation_strings":["Tsinghua University,BNRist,Beijing,China","BNRist, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,BNRist,Beijing,China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062141045","display_name":"Lei Wang","orcid":"https://orcid.org/0000-0002-6145-0705"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Wang","raw_affiliation_strings":["Tsinghua University,BNRist,Beijing,China","BNRist, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,BNRist,Beijing,China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110193522","display_name":"Hao Sun","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Sun","raw_affiliation_strings":["Tsinghua University,BNRist,Beijing,China","BNRist, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,BNRist,Beijing,China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084439869","display_name":"Jingchuan Wei","orcid":"https://orcid.org/0000-0001-5869-9094"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingchuan Wei","raw_affiliation_strings":["Tsinghua University,BNRist,Beijing,China","BNRist, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,BNRist,Beijing,China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100358856","display_name":"Leibo Liu","orcid":"https://orcid.org/0000-0001-7548-4116"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Leibo Liu","raw_affiliation_strings":["Tsinghua University,BNRist,Beijing,China","BNRist, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,BNRist,Beijing,China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036023084","display_name":"Shaojun Wei","orcid":"https://orcid.org/0000-0001-5117-7920"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaojun Wei","raw_affiliation_strings":["Tsinghua University,BNRist,Beijing,China","BNRist, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,BNRist,Beijing,China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059343235","display_name":"Yang Hu","orcid":"https://orcid.org/0000-0003-2388-7719"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yang Hu","raw_affiliation_strings":["Tsinghua University,BNRist,Beijing,China","BNRist, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,BNRist,Beijing,China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5054524841","display_name":"Shouyi Yin","orcid":"https://orcid.org/0000-0003-2309-572X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shouyi Yin","raw_affiliation_strings":["Tsinghua University,BNRist,Beijing,China","BNRist, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,BNRist,Beijing,China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"3"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9635999798774719,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9635999798774719,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12326","display_name":"Network Packet Processing and Optimization","score":0.9412000179290771,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.941100001335144,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.5186883211135864},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.3657471239566803},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3657117187976837},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.35582780838012695},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3557049632072449},{"id":"https://openalex.org/keywords/combinatorics","display_name":"Combinatorics","score":0.33925768733024597},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1641882359981537},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.05625414848327637}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5186883211135864},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.3657471239566803},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3657117187976837},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.35582780838012695},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3557049632072449},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.33925768733024597},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1641882359981537},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.05625414848327637}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/a-sscc58667.2023.10347930","is_oa":false,"landing_page_url":"https://doi.org/10.1109/a-sscc58667.2023.10347930","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Asian Solid-State Circuits Conference (A-SSCC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.44999998807907104}],"awards":[{"id":"https://openalex.org/G7028138989","display_name":null,"funder_award_id":"2021ZD0114400","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G975256117","display_name":null,"funder_award_id":"U19B2041,62125403,92164301","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":6,"referenced_works":["https://openalex.org/W3135835558","https://openalex.org/W3168124404","https://openalex.org/W3184748710","https://openalex.org/W4220865834","https://openalex.org/W4312081748","https://openalex.org/W6796494063"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2935759653","https://openalex.org/W3105167352","https://openalex.org/W54078636","https://openalex.org/W2954470139","https://openalex.org/W1501425562","https://openalex.org/W2902782467","https://openalex.org/W3084825885","https://openalex.org/W2298861036","https://openalex.org/W2271181815"],"abstract_inverted_index":{"Transformer":[0],"models":[1],"shows":[2,98],"state-of-the-art":[3],"results":[4],"in":[5,118,124,261,357],"natural":[6],"language":[7],"processing":[8],"and":[9,38,51,63,89,109,258,355],"computer":[10],"vision,":[11],"leveraging":[12],"a":[13,220,227,265,314],"multi-headed":[14],"self-attention":[15],"mechanism.":[16],"In":[17,342],"each":[18],"head,":[19],"the":[20,125,145,173,186,190,206,209,253,283,300,318,322,343],"operation":[21],"is":[22,73,147,249,278,351],"defined":[23],"as":[24,310],"<tex":[25,32,39,56,60,64,69,128,244,267,285],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[26,33,40,57,61,65,70,129,245,268,286],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$\\text{Attn}=\\text{Softmax}(\\mathrm{Q}\\cdot":[27],"\\mathrm{K}^{\\top})\\cdot":[28],"\\mathrm{V}$</tex>":[29],",":[30,59,67],"where":[31],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$\\mathrm{Q}=\\mathrm{X}\\cdot":[34],"\\mathrm{W}_{\\mathrm{Q}},\\":[35],"\\mathrm{K}=\\mathrm{X}\\cdot":[36],"\\mathrm{W}_{\\mathrm{K}}$</tex>":[37],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$\\mathrm{V}=\\mathrm{X}\\cdot":[41],"\\mathrm{W}_{\\mathrm{V}}$</tex>":[42],"are":[43,111,155,166],"linear":[44],"transformations":[45],"for":[46,230,313,340],"Query":[47],"(Q),":[48],"Key":[49],"(K)":[50],"Value":[52],"(V)":[53],"with":[54,141,180,234,272,289,302,325,337],"weight":[55,349],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$\\mathrm{W}_{\\mathrm{Q}}$</tex>":[58],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$\\mathrm{W}_{\\mathrm{K}}$</tex>":[62],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$\\mathrm{W}_{\\mathrm{V}}$</tex>":[66],"respectively.":[68],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$\\mathrm{Q}\\cdot":[71,130,246,287],"\\mathrm{K}^{\\top}$</tex>":[72,131,247,288],"responsible":[74],"to":[75,159,168,215,237,251,280],"learn":[76],"relevance":[77],"scores":[78,154],"between":[79],"tokens":[80],"(X).":[81],"Previous":[82],"CIM":[83,119,135,228,262,315,331,345],"chips":[84],"faces":[85],"new":[86,217],"challenges":[87],"within":[88],"across":[90],"attention":[91,153],"heads":[92],"(Fig.":[93],"1):":[94],"(1)":[95,241],"Computing-in-memory":[96],"(CIM)":[97],"great":[99],"advantages":[100],"only":[101,177],"if":[102],"fixing":[103],"pre-trained":[104],"weights.":[105],"However,":[106],"since":[107],"Q":[108],"K":[110,117],"both":[112],"generated":[113],"at":[114],"runtime,":[115],"loading":[116,254],"macros":[120,213],"consumes":[121],"more":[122],"energy":[123],"computing":[126,187],"of":[127,164,175,189,193,201,208,255,321],".":[132],"(2)":[133,292],"A":[134,242,293,329],"macro":[136,346],"performs":[137],"multiply-accumulate":[138],"(MAC)":[139],"operations":[140],"bit-serial":[142],"inputs,":[143],"so":[144],"latency":[146,188,320],"determined":[148],"by":[149,157],"input":[150],"precisions.":[151],"The":[152],"normalized":[156],"Softmax":[158],"Probability":[160],"(P).":[161],"Most":[162],"elements":[163,179,301,308],"P":[165],"close":[167],"or":[169],"exactly":[170],"zero.":[171],"For":[172],"P.V":[174],"Bert-T,":[176],"10%":[178],"large":[181,303],"effective":[182],"bit-width":[183],"(EBW)":[184],"exacerbate":[185],"rest":[191,323],"90%":[192],"inputs.":[194],"(3)":[195,328],"On-chip":[196],"SRAM-CIM":[197],"cannot":[198],"hold":[199],"weights":[200],"all":[202,212],"heads.":[203],"As":[204],"completing":[205],"computation":[207],"stored":[210],"weights,":[211,218],"need":[214],"reload":[216],"causing":[219],"significant":[221],"performance":[222],"loss.":[223],"This":[224],"work":[225],"designs":[226],"processor":[229],"Transformer,":[231],"called":[232,305],"CIMFormer,":[233],"three":[235],"features":[236],"solve":[238],"above":[239],"challenges:":[240],"token-slimmed":[243],"reformulation":[248],"proposed":[250],"reduce":[252],"intermediate":[256],"data":[257],"redundant":[259],"computations":[260],"macros.":[263],"Besides,":[264],"column-partitioned":[266],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$\\mathrm{X}\\vert":[269],"\\mathrm{W}-":[270],"\\text{CIM}$</tex>":[271],"flexible":[273],"set-aggregate":[274],"adder":[275],"tree":[276],"(Flex-AT)":[277],"designed":[279],"efficiently":[281],"match":[282],"reformulated":[284],"high":[290],"utilization.":[291],"principal":[294,306],"possibility":[295,307],"gather-scatter":[296],"scheduler":[297],"(PPGSS)":[298],"collects":[299],"EBWs,":[304],"(PPEs),":[309],"simultaneous":[311],"activations":[312,324],"macro,":[316],"reducing":[317],"compute":[319],"small":[326],"EBWs.":[327],"systolic":[330,344,359],"array":[332],"supports":[333],"bidirectional":[334],"matrix":[335],"multiplications":[336],"macro-interleaved":[338],"broadcast":[339],"activations.":[341],"array,":[347],"array-level":[348],"reloading":[350],"divided":[352],"into":[353],"macro-level":[354,358],"hidden":[356],"computing.":[360]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}