{"id":"https://openalex.org/W7171722648","doi":"https://doi.org/10.48550/arxiv.2607.24953","title":"Stable FP4 Training via Transposition-Invariant Block Quantization","display_name":"Stable FP4 Training via Transposition-Invariant Block Quantization","publication_year":2026,"publication_date":"2026-07-27","ids":{"openalex":"https://openalex.org/W7171722648","doi":"https://doi.org/10.48550/arxiv.2607.24953"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.24953","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.24953","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.24953","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133551656","display_name":"Mehdi Rahimifar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rahimifar, Mehdi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5143976503","display_name":"Amin Darabi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Darabi, Amin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094139081","display_name":"Mehran Taghian Jazi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jazi, Mehran Taghian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5143924857","display_name":"Xing Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Xing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5143872963","display_name":"Yao Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5143960514","display_name":"Zhijun Tu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tu, Zhijun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5143927228","display_name":"Yufei Cui","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cui, Yufei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086399135","display_name":"Yunke Peng","orcid":"https://orcid.org/0000-0001-5240-0725"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Yunke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5143901998","display_name":"Hongliang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Hongliang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.27970001101493835,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.27970001101493835,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.13979999721050262,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.12800000607967377,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.7627000212669373},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.7426999807357788},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.6823999881744385},{"id":"https://openalex.org/keywords/rounding","display_name":"Rounding","score":0.545199990272522},{"id":"https://openalex.org/keywords/block","display_name":"Block (permutation group theory)","score":0.49900001287460327},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.4413999915122986}],"concepts":[{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.7627000212669373},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.7426999807357788},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6823999881744385},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6234999895095825},{"id":"https://openalex.org/C136625980","wikidata":"https://www.wikidata.org/wiki/Q663208","display_name":"Rounding","level":2,"score":0.545199990272522},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.49900001287460327},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4717999994754791},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.4413999915122986},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.39430001378059387},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.33559998869895935},{"id":"https://openalex.org/C2776639698","wikidata":"https://www.wikidata.org/wiki/Q4927140","display_name":"Block Error Rate","level":3,"score":0.31470000743865967},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.2980000078678131},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.2957000136375427},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.2743000090122223},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.27320000529289246},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2587999999523163}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.24953","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.24953","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.24953","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.24953","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reducing":[0],"training":[1,87,177,208],"precision":[2],"is":[3,201],"a":[4,36,85,142,158,212],"key":[5,139],"lever":[6],"for":[7,136],"improving":[8],"the":[9,67,127],"e":[10,215,220],"ciency":[11],"of":[12,39,129],"large":[13],"language":[14],"model":[15],"(LLM)":[16],"training,":[17],"but":[18],"pushing":[19],"beyond":[20],"FP8":[21],"to":[22,30,66,73,117,154,165,204],"4-bit":[23],"oating":[24],"point":[25],"(FP4)":[26],"remains":[27],"challenging":[28],"due":[29],"instability":[31,41],"during":[32],"optimization.":[33],"We":[34,107,146],"identify":[35],"fundamental":[37],"source":[38],"this":[40,81,110],"in":[42,188],"existing":[43],"microscaling":[44],"approaches:":[45],"scale":[46],"inconsistency":[47],"induced":[48],"by":[49],"tensor":[50],"transposition.":[51],"In":[52],"conventional":[53],"1D":[54],"block":[55,92],"quantization,":[56,94],"forward":[57,103],"and":[58,75,99,104,114,121,138,157,178,190,214],"backward":[59,105],"passes":[60],"assign":[61],"di":[62],"erent":[63],"scaling":[64,98,113,199],"factors":[65],"same":[68],"values":[69],"after":[70],"transposition,":[71],"leading":[72],"biased":[74],"unstable":[76],"gradient":[77],"updates.":[78],"To":[79,125],"address":[80],"issue,":[82],"we":[83,132],"propose":[84],"low-precision":[86],"framework":[88],"based":[89],"on":[90,150,163],"2D":[91],"FP4":[93,176,207],"which":[95],"enforces":[96],"transposition-invariant":[97],"preserves":[100],"consistency":[101,200],"between":[102],"computations.":[106],"further":[108],"combine":[109],"with":[111,183],"truncation-free":[112],"stochastic":[115],"rounding":[116],"control":[118],"quantization":[119,135],"error":[120],"maintain":[122],"unbiased":[123],"gradients.":[124],"handle":[126],"sensitivity":[128],"attention":[130],"mechanisms,":[131],"adopt":[133],"MXFP8":[134],"query":[137],"projections,":[140],"yielding":[141],"practical":[143,206],"mixed-precision":[144],"design.":[145],"evaluate":[147],"our":[148,171],"method":[149],"dense":[151],"LLMs":[152],"up":[153,164],"7B":[155],"parameters":[156],"30B":[159],"Mixture-of-Experts":[160],"model,":[161],"trained":[162],"100B":[166],"tokens.":[167],"Across":[168],"all":[169],"settings,":[170],"approach":[172],"achieves":[173],"stable":[174],"end-to-end":[175],"closely":[179],"matches":[180],"BF16":[181],"performance,":[182],"less":[184],"than":[185],"1.3%":[186],"degradation":[187],"perplexity":[189],"downstream":[191],"accuracy.":[192],"These":[193],"results":[194],"demonstrate":[195],"that":[196],"enforcing":[197],"forwardbackward":[198],"su":[202],"cient":[203,221],"enable":[205],"at":[209],"scale,":[210],"providing":[211],"simple":[213],"ective":[216],"pathway":[217],"toward":[218],"more":[219],"LLM":[222],"training.":[223]},"counts_by_year":[],"updated_date":"2026-07-30T17:31:21.811387","created_date":"2026-07-30T00:00:00"}