{"id":"https://openalex.org/W7164013322","doi":"https://doi.org/10.48550/arxiv.2606.08761","title":"APEX4: Efficient Pure W4A4 LLM Inference via Intra-SM Compute Rebalancing","display_name":"APEX4: Efficient Pure W4A4 LLM Inference via Intra-SM Compute Rebalancing","publication_year":2026,"publication_date":"2026-06-07","ids":{"openalex":"https://openalex.org/W7164013322","doi":"https://doi.org/10.48550/arxiv.2606.08761"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.08761","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08761","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.08761","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138250220","display_name":"Hong Quan Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Hong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043579361","display_name":"Nianhui Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Nianhui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138240897","display_name":"Weixing Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Weixing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053166615","display_name":"Jona Otholt","orcid":"https://orcid.org/0009-0005-1071-1230"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Otholt, Jona","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138246745","display_name":"Christoph Meinel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Meinel, Christoph","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5101039243","display_name":"Haojin Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Haojin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.4542999863624573,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.4542999863624573,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.1501999944448471,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.04450000077486038,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.9071000218391418},{"id":"https://openalex.org/keywords/cuda","display_name":"CUDA","score":0.8482000231742859},{"id":"https://openalex.org/keywords/granularity","display_name":"Granularity","score":0.6665999889373779},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.5335999727249146},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5170999765396118},{"id":"https://openalex.org/keywords/general-purpose-computing-on-graphics-processing-units","display_name":"General-purpose computing on graphics processing units","score":0.507099986076355},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.46650001406669617},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.46320000290870667},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.43149998784065247}],"concepts":[{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.9071000218391418},{"id":"https://openalex.org/C2778119891","wikidata":"https://www.wikidata.org/wiki/Q477690","display_name":"CUDA","level":2,"score":0.8482000231742859},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7271000146865845},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.6665999889373779},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6629999876022339},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.5335999727249146},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5170999765396118},{"id":"https://openalex.org/C50630238","wikidata":"https://www.wikidata.org/wiki/Q971505","display_name":"General-purpose computing on graphics processing units","level":3,"score":0.507099986076355},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.46650001406669617},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.46320000290870667},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.43149998784065247},{"id":"https://openalex.org/C459310","wikidata":"https://www.wikidata.org/wiki/Q117801","display_name":"Computational science","level":1,"score":0.4171999990940094},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.41530001163482666},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.40560001134872437},{"id":"https://openalex.org/C155281189","wikidata":"https://www.wikidata.org/wiki/Q3518150","display_name":"Tensor (intrinsic definition)","level":2,"score":0.3677000105381012},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.33660000562667847},{"id":"https://openalex.org/C24881265","wikidata":"https://www.wikidata.org/wiki/Q757267","display_name":"Voronoi diagram","level":2,"score":0.3151000142097473},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3027999997138977},{"id":"https://openalex.org/C2781311116","wikidata":"https://www.wikidata.org/wiki/Q83306","display_name":"Group (periodic table)","level":2,"score":0.29789999127388},{"id":"https://openalex.org/C205711294","wikidata":"https://www.wikidata.org/wiki/Q176953","display_name":"Rendering (computer graphics)","level":2,"score":0.296999990940094},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.2838999927043915},{"id":"https://openalex.org/C42935608","wikidata":"https://www.wikidata.org/wiki/Q190411","display_name":"Field-programmable gate array","level":2,"score":0.2825999855995178},{"id":"https://openalex.org/C147297375","wikidata":"https://www.wikidata.org/wiki/Q6674930","display_name":"Look-ahead","level":2,"score":0.2815999984741211},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.2802000045776367},{"id":"https://openalex.org/C2778915421","wikidata":"https://www.wikidata.org/wiki/Q3643177","display_name":"Performance improvement","level":2,"score":0.2689000070095062},{"id":"https://openalex.org/C32653426","wikidata":"https://www.wikidata.org/wiki/Q3813641","display_name":"Background subtraction","level":3,"score":0.26660001277923584},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.25940001010894775},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2517000138759613}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.08761","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08761","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.08761","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08761","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"W4A4":[0,84],"quantization":[1],"promises":[2],"full":[3],"utilization":[4],"of":[5,29,121],"INT4":[6,102],"Tensor":[7,51],"Cores,":[8],"yet":[9,73],"group":[10],"dequantization":[11,114],"overhead":[12],"on":[13,70,77,123,149,154,158],"CUDA":[14,54,112],"Cores":[15,52,55,113],"has":[16],"driven":[17],"existing":[18],"systems":[19],"to":[20,53,75,109,145,165],"mixed-precision":[21],"fallbacks.":[22],"We":[23],"present":[24],"the":[25,50,60,64,111,168],"first":[26],"systematic":[27],"study":[28],"how":[30],"intra-SM":[31],"compute":[32],"balance":[33],"governs":[34],"this":[35,94],"bottleneck.":[36,115],"Through":[37],"controlled":[38],"benchmarks":[39],"across":[40],"four":[41],"GPUs":[42],"from":[43],"Ampere":[44],"and":[45,125,152],"Ada":[46],"architectures,":[47],"we":[48,96],"identify":[49],"throughput":[56],"ratio":[57],"($\u03c1$)":[58],"as":[59,86,135],"primary":[61],"hardware":[62],"indicator:":[63],"W4A4-g128":[65],"kernel":[66],"yields":[67],"$2.0$--$2.5\\times$":[68],"speedup":[69,148],"RTX~3090":[71,155],"($\u03c1=16$)":[72],"degrades":[74],"$0.43$--$0.47\\times$":[76],"A100":[78,163],"($\u03c1=64$)":[79,164],"in":[80,131,139],"compute-bond":[81],"scenarios,":[82],"establishing":[83],"viability":[85],"platform-dependent":[87],"rather":[88],"than":[89],"universally":[90],"infeasible.":[91],"Guided":[92],"by":[93,129],"finding,":[95],"build":[97],"\\textbf{APEX4},":[98],"which":[99],"co-designs":[100],"pure":[101],"GEMM":[103],"kernels":[104],"with":[105],"$\u03c1$-aware":[106],"granularity":[107],"adaptation":[108],"mitigate":[110],"APEX4":[116],"achieves":[117],"perplexity":[118],"within":[119],"0.63":[120],"FP16":[122],"LLaMA-2-70B":[124],"outperforms":[126],"W4Ax":[127],"Atom-g128":[128],"4.0\\%--4.4\\%":[130],"zero-shot":[132],"accuracy.":[133],"Deployed":[134],"a":[136],"drop-in":[137],"replacement":[138],"unmodified":[140],"vLLM,":[141],"it":[142],"delivers":[143],"up":[144],"$1.66\\times$":[146],"end-to-end":[147],"L40S":[150],"($\u03c1=8$),":[151],"$1.78\\times$":[153],"($\u03c1=16$),":[156,160],"$2.09\\times$":[157],"A40":[159],"while":[161],"recovering":[162],"$1.20$--$1.40\\times$":[166],"via":[167],"mixed-granularity":[169],"mode.":[170],"Our":[171],"code":[172],"is":[173],"available":[174],"at":[175],"https://github.com/APEX4-W4A4/APEX4-W4A4.":[176]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-10T00:00:00"}