{"id":"https://openalex.org/W7131882702","doi":"https://doi.org/10.48550/arxiv.2602.23200","title":"InnerQ: Hardware-Aware Tuning-Free Quantization of KV Cache for Large Language Models","display_name":"InnerQ: Hardware-Aware Tuning-Free Quantization of KV Cache for Large Language Models","publication_year":2026,"publication_date":"2026-02-26","ids":{"openalex":"https://openalex.org/W7131882702","doi":"https://doi.org/10.48550/arxiv.2602.23200"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.23200","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052444966","display_name":"Seyed Mohammad Sadegh Hosseini","orcid":"https://orcid.org/0000-0003-2534-7193"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hosseini, Sayed Mohammadreza Tayaranian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057939934","display_name":"Amir Ardakani","orcid":"https://orcid.org/0000-0002-3359-1180"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ardakani, Amir","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5091600002","display_name":"Warren J. Gross","orcid":"https://orcid.org/0000-0002-6226-6037"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gross, Warren J.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.18272955,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.4377000033855438,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.4377000033855438,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.09520000219345093,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.0737999975681305,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.6945000290870667},{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.6460999846458435},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.5820000171661377},{"id":"https://openalex.org/keywords/memory-footprint","display_name":"Memory footprint","score":0.5020999908447266},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4587000012397766},{"id":"https://openalex.org/keywords/reuse","display_name":"Reuse","score":0.43709999322891235},{"id":"https://openalex.org/keywords/cpu-cache","display_name":"CPU cache","score":0.42419999837875366},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.3874000012874603}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8126999735832214},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.6945000290870667},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.6460999846458435},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.5820000171661377},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5354999899864197},{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.5020999908447266},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.49799999594688416},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4587000012397766},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.43709999322891235},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.42419999837875366},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.3874000012874603},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.37040001153945923},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.33970001339912415},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.29260000586509705},{"id":"https://openalex.org/C17349429","wikidata":"https://www.wikidata.org/wiki/Q1049914","display_name":"Matrix multiplication","level":3,"score":0.29089999198913574},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2897000014781952},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.26570001244544983},{"id":"https://openalex.org/C56951928","wikidata":"https://www.wikidata.org/wiki/Q3539213","display_name":"Trimming","level":2,"score":0.26460000872612},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.25920000672340393},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.2583000063896179},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.2547999918460846}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.23200","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.23200","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.23200","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.23200","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"When":[0],"transformer-based":[1],"language":[2],"models":[3,222],"are":[4,23],"deployed":[5],"for":[6,37,170,180],"text":[7],"generation,":[8],"most":[9],"of":[10,30,77,196],"the":[11,17,27,45,58,70,148,197,207],"inference":[12],"time":[13],"is":[14,34,44],"spent":[15],"in":[16],"decoding":[18,32],"stage,":[19],"where":[20],"output":[21],"tokens":[22,183,187],"generated":[24],"sequentially.":[25],"Reducing":[26],"hardware":[28],"cost":[29],"each":[31,171],"step":[33],"therefore":[35],"critical":[36],"efficient":[38],"long-context":[39],"generation.":[40],"A":[41],"major":[42],"bottleneck":[43],"key-value":[46],"(KV)":[47],"cache,":[48,199],"whose":[49],"size":[50],"grows":[51],"with":[52,113],"sequence":[53],"length":[54],"and":[55,116,131,145,184,192,204,220],"often":[56],"dominates":[57],"model's":[59],"memory":[60,129],"footprint.":[61],"Prior":[62],"work":[63],"has":[64],"proposed":[65],"quantization":[66,86,99,143,169,236],"methods":[67,144],"to":[68,188,210,232],"compress":[69],"KV":[71,84,141,234],"cache":[72,85,102,142,235],"while":[73],"minimizing":[74],"its":[75],"loss":[76],"precision.":[78],"We":[79],"present":[80],"InnerQ,":[81],"a":[82,125],"hardware-aware":[83],"scheme":[87],"that":[88,224],"reduces":[89,128],"decode":[90],"latency":[91],"without":[92],"compromising":[93],"evaluation":[94,229],"performance.":[95],"InnerQ":[96,127,157,225],"performs":[97],"group-wise":[98],"by":[100],"grouping":[101,109],"matrices":[103],"along":[104],"their":[105],"inner":[106],"dimension.":[107],"This":[108],"strategy":[110],"aligns":[111],"dequantization":[112],"vector-matrix":[114],"multiplication":[115],"increases":[117],"data":[118],"reuse":[119],"across":[120],"GPU":[121],"compute":[122],"units.":[123],"As":[124],"result,":[126],"access":[130],"accelerates":[132],"dequantization,":[133],"achieving":[134],"an":[135],"average":[136],"$1.3\\times$":[137],"speedup":[138],"over":[139,147],"prior":[140,233],"$2.7\\times$":[146],"non-quantized":[149],"baseline.":[150],"To":[151],"maintain":[152],"fidelity":[153],"under":[154],"aggressive":[155],"compression,":[156],"incorporates":[158],"three":[159],"techniques:":[160],"(i)":[161],"hybrid":[162],"quantization,":[163],"which":[164],"chooses":[165],"symmetric":[166],"or":[167],"asymmetric":[168],"group":[172],"based":[173],"on":[174,218],"local":[175],"statistics;":[176],"(ii)":[177],"high-precision":[178],"windows":[179],"both":[181],"recent":[182],"attention":[185],"sink":[186],"mitigate":[189],"outlier":[190],"leakage;":[191],"(iii)":[193],"per-channel":[194],"normalization":[195],"key":[198],"computed":[200],"once":[201],"during":[202],"prefill":[203],"folded":[205],"into":[206],"model":[208],"parameters":[209],"eliminate":[211],"runtime":[212],"overhead.":[213],"Beyond":[214],"reducing":[215],"latency,":[216],"experiments":[217],"Llama":[219],"Mistral":[221],"show":[223],"also":[226],"improves":[227],"few-shot":[228],"scores":[230],"relative":[231],"methods.":[237]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2026-02-28T00:00:00"}