{"id":"https://openalex.org/W4409282394","doi":"https://doi.org/10.1145/3676536.3676776","title":"Towards Floating Point-Based Attention-Free LLM: Hybrid PIM with Non-Uniform Data Format and Reduced Multiplications","display_name":"Towards Floating Point-Based Attention-Free LLM: Hybrid PIM with Non-Uniform Data Format and Reduced Multiplications","publication_year":2024,"publication_date":"2024-10-27","ids":{"openalex":"https://openalex.org/W4409282394","doi":"https://doi.org/10.1145/3676536.3676776"},"language":"en","primary_location":{"id":"doi:10.1145/3676536.3676776","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3676536.3676776","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3676536.3676776","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd IEEE/ACM International Conference on Computer-Aided Design","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3676536.3676776","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100297509","display_name":"Lidong Guo","orcid":"https://orcid.org/0000-0003-4162-6360"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lidong Guo","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-4162-6360","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103138440","display_name":"Zhenhua Zhu","orcid":"https://orcid.org/0009-0007-9259-7180"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhenhua Zhu","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-9259-7180","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083660349","display_name":"Tengxuan Liu","orcid":"https://orcid.org/0009-0007-8943-1577"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tengxuan Liu","raw_affiliation_strings":["Infinigence-AI, Beijing, China","Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-8943-1577","affiliations":[{"raw_affiliation_string":"Infinigence-AI, Beijing, China","institution_ids":[]},{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086217226","display_name":"Xuefei Ning","orcid":"https://orcid.org/0000-0003-2209-8312"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuefei Ning","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-2209-8312","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005872704","display_name":"Shiyao Li","orcid":"https://orcid.org/0009-0007-4755-6881"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shiyao Li","raw_affiliation_strings":["Infinigence-AI, Beijing, China","Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-4755-6881","affiliations":[{"raw_affiliation_string":"Infinigence-AI, Beijing, China","institution_ids":[]},{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015946486","display_name":"Guohao Dai","orcid":"https://orcid.org/0000-0003-0849-3252"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guohao Dai","raw_affiliation_strings":["Infinigence-AI, Shanghai, China","Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-0849-3252","affiliations":[{"raw_affiliation_string":"Infinigence-AI, Shanghai, China","institution_ids":[]},{"raw_affiliation_string":"Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023755254","display_name":"Huazhong Yang","orcid":"https://orcid.org/0000-0003-2421-353X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huazhong Yang","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-2421-353X","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054651245","display_name":"Wangyang Fu","orcid":"https://orcid.org/0000-0003-3185-0457"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wangyang Fu","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3185-0457","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100445061","display_name":"Yu Wang","orcid":"https://orcid.org/0000-0001-6108-5157"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Wang","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6108-5157","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11206","display_name":"Model Reduction and Neural Networks","score":0.9593999981880188,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11206","display_name":"Model Reduction and Neural Networks","score":0.9593999981880188,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12692","display_name":"Magnetic Field Sensors Techniques","score":0.9420999884605408,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11263","display_name":"Electromagnetic Simulation and Numerical Methods","score":0.9211000204086304,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6183119416236877},{"id":"https://openalex.org/keywords/floating-point","display_name":"Floating point","score":0.5704531073570251},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.5003612041473389},{"id":"https://openalex.org/keywords/arithmetic","display_name":"Arithmetic","score":0.4742635488510132},{"id":"https://openalex.org/keywords/fixed-point-arithmetic","display_name":"Fixed-point arithmetic","score":0.4204178750514984},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.3968321681022644},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.29033970832824707},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.18758150935173035}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6183119416236877},{"id":"https://openalex.org/C84211073","wikidata":"https://www.wikidata.org/wiki/Q117879","display_name":"Floating point","level":2,"score":0.5704531073570251},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.5003612041473389},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.4742635488510132},{"id":"https://openalex.org/C163973906","wikidata":"https://www.wikidata.org/wiki/Q649900","display_name":"Fixed-point arithmetic","level":3,"score":0.4204178750514984},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3968321681022644},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.29033970832824707},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.18758150935173035},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3676536.3676776","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3676536.3676776","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3676536.3676776","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd IEEE/ACM International Conference on Computer-Aided Design","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3676536.3676776","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3676536.3676776","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3676536.3676776","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd IEEE/ACM International Conference on Computer-Aided Design","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2045798616","display_name":null,"funder_award_id":"62325405","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G663016133","display_name":null,"funder_award_id":"62204164","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6803541034","display_name":null,"funder_award_id":"U21B2031","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8262591375","display_name":null,"funder_award_id":"62104128","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320329777","display_name":"Beijing National Research Center For Information Science And Technology","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4409282394.pdf","grobid_xml":"https://content.openalex.org/works/W4409282394.grobid-xml"},"referenced_works_count":16,"referenced_works":["https://openalex.org/W2105102111","https://openalex.org/W2508602506","https://openalex.org/W2518281301","https://openalex.org/W2911910392","https://openalex.org/W2946609015","https://openalex.org/W2946659370","https://openalex.org/W2998617917","https://openalex.org/W3139521791","https://openalex.org/W3176145847","https://openalex.org/W3194676777","https://openalex.org/W4200404159","https://openalex.org/W4313332258","https://openalex.org/W4323022446","https://openalex.org/W4323338529","https://openalex.org/W4377079683","https://openalex.org/W4408183178"],"related_works":["https://openalex.org/W4255129386","https://openalex.org/W1988006935","https://openalex.org/W1585685783","https://openalex.org/W3206224488","https://openalex.org/W2119414612","https://openalex.org/W2268486090","https://openalex.org/W2288960809","https://openalex.org/W1935080020","https://openalex.org/W4245068234","https://openalex.org/W1978352992"],"abstract_inverted_index":{"Attention-free":[0],"large":[1],"language":[2],"models":[3,29,72],"(LLMs),":[4],"such":[5],"as":[6,12],"Mamba":[7],"and":[8,36,63,88,119],"RWKV,":[9],"have":[10,46],"emerged":[11],"promising":[13],"architectures":[14,45,69,80,110],"to":[15,50,97,115],"address":[16],"the":[17,52,59,91,98,120,125,141],"quadratic":[18],"attention":[19],"complexity":[20],"of":[21,27,128],"Transformer":[22],"models.":[23],"The":[24,40],"inference":[25],"bottleneck":[26],"these":[28],"lies":[30],"in":[31,90,148],"memory-bound":[32],"matrix-vector":[33],"multiplications":[34,38],"(MVMs)":[35],"element-wise":[37],"(EWMs).":[39],"emerging":[41],"RRAM/SRAM-based":[42],"Processing-In-Memory":[43],"(PIM)":[44],"shown":[47],"great":[48],"potential":[49],"overcome":[51],"memory":[53,143],"wall":[54],"problem.":[55],"However,":[56],"constrained":[57],"by":[58],"supported":[60],"data":[61,104],"format":[62],"operator":[64],"type,":[65],"directly":[66],"adopting":[67],"PIM":[68,79,109,135],"for":[70],"attention-free":[71],"faces":[73],"three":[74],"challenges:":[75],"(1)":[76],"RRAM-based":[77],"analog":[78,92],"perform":[81],"integer":[82],"(INT)":[83],"MVMs":[84],"using":[85,134],"voltage,":[86],"current,":[87],"conductance":[89],"domain,":[93],"limiting":[94],"their":[95],"application":[96],"more":[99],"accurate":[100],"floating":[101],"point":[102],"(FP)":[103],"format;":[105],"(2)":[106],"SRAM-based":[107],"digital":[108],"require":[111],"additional":[112],"decoder":[113],"circuits":[114],"support":[116],"FP":[117],"format,":[118],"SRAM":[121],"capacity":[122],"cannot":[123],"satisfy":[124],"storage":[126],"requirement":[127],"LLMs;":[129],"(3)":[130],"When":[131],"performing":[132],"EWMs":[133],"architectures,":[136],"only":[137],"one":[138],"row/column":[139],"or":[140],"diagonal":[142],"cells":[144],"are":[145],"activated,":[146],"resulting":[147],"severe":[149],"device":[150],"under-utilization.":[151]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}