{"id":"https://openalex.org/W7161697515","doi":"https://doi.org/10.48550/arxiv.2605.17471","title":"WinQ: Accelerating Quantization-Aware Training of Language Models Around Saddle Points","display_name":"WinQ: Accelerating Quantization-Aware Training of Language Models Around Saddle Points","publication_year":2026,"publication_date":"2026-05-17","ids":{"openalex":"https://openalex.org/W7161697515","doi":"https://doi.org/10.48550/arxiv.2605.17471"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.17471","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.17471","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.17471","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136464324","display_name":"Dongyue Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Dongyue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136495492","display_name":"Zechun Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Zechun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136472382","display_name":"Kai Yi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yi, Kai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026741810","display_name":"Zhenshuo Zhang","orcid":"https://orcid.org/0000-0001-6878-097X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Zhenshuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136459854","display_name":"Changsheng Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Changsheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136498862","display_name":"Raghuraman Krishnamoorthi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krishnamoorthi, Raghuraman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008858683","display_name":"Harshit Khaitan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Khaitan, Harshit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136460185","display_name":"Hongyang R. Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Hongyang R.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136487759","display_name":"Steven Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Steven","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2468000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2468000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.15530000627040863,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.12060000002384186,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hessian-matrix","display_name":"Hessian matrix","score":0.7918999791145325},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.7551000118255615},{"id":"https://openalex.org/keywords/eigenvalues-and-eigenvectors","display_name":"Eigenvalues and eigenvectors","score":0.6802999973297119},{"id":"https://openalex.org/keywords/interpolation","display_name":"Interpolation (computer graphics)","score":0.44040000438690186},{"id":"https://openalex.org/keywords/saddle-point","display_name":"Saddle point","score":0.43959999084472656},{"id":"https://openalex.org/keywords/saddle","display_name":"Saddle","score":0.4180000126361847},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.4104999899864197},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.40790000557899475}],"concepts":[{"id":"https://openalex.org/C203616005","wikidata":"https://www.wikidata.org/wiki/Q620495","display_name":"Hessian matrix","level":2,"score":0.7918999791145325},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.7551000118255615},{"id":"https://openalex.org/C158693339","wikidata":"https://www.wikidata.org/wiki/Q190524","display_name":"Eigenvalues and eigenvectors","level":2,"score":0.6802999973297119},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5575000047683716},{"id":"https://openalex.org/C137800194","wikidata":"https://www.wikidata.org/wiki/Q11713455","display_name":"Interpolation (computer graphics)","level":3,"score":0.44040000438690186},{"id":"https://openalex.org/C2681867","wikidata":"https://www.wikidata.org/wiki/Q690935","display_name":"Saddle point","level":2,"score":0.43959999084472656},{"id":"https://openalex.org/C2777127463","wikidata":"https://www.wikidata.org/wiki/Q10862618","display_name":"Saddle","level":2,"score":0.4180000126361847},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.4104999899864197},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.40790000557899475},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.399399995803833},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.3828999996185303},{"id":"https://openalex.org/C149629883","wikidata":"https://www.wikidata.org/wiki/Q660926","display_name":"Fraction (chemistry)","level":2,"score":0.36149999499320984},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.34130001068115234},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.3285999894142151},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3212999999523163},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.314300000667572},{"id":"https://openalex.org/C171836373","wikidata":"https://www.wikidata.org/wiki/Q2266329","display_name":"Linear interpolation","level":3,"score":0.3077999949455261},{"id":"https://openalex.org/C2780129190","wikidata":"https://www.wikidata.org/wiki/Q5071912","display_name":"Changeover","level":3,"score":0.30329999327659607},{"id":"https://openalex.org/C187691185","wikidata":"https://www.wikidata.org/wiki/Q2020720","display_name":"Grid","level":2,"score":0.29660001397132874},{"id":"https://openalex.org/C199833920","wikidata":"https://www.wikidata.org/wiki/Q612536","display_name":"Vector quantization","level":2,"score":0.27160000801086426},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.25380000472068787}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.17471","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.17471","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.17471","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.17471","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Quantization-aware":[0],"training":[1,11,189],"(QAT)":[2],"is":[3,23,115],"widely":[4],"adopted":[5],"to":[6,71,128,137,149,164,177,198],"quantize":[7],"language":[8,209],"models":[9],"by":[10,57,175,196],"full-precision":[12,142],"weights":[13,69,136,163],"using":[14],"gradients":[15,160],"from":[16],"the":[17,53,59,62,68,82,107,112,138,147,150,166,187],"quantized":[18,144],"model.":[19],"The":[20],"main":[21],"bottleneck":[22],"its":[24,43],"slow":[25],"convergence":[26,54],"and":[27,88,143,153,157,184,213],"early":[28],"performance":[29],"plateau,":[30],"particularly":[31],"below":[32],"4-bit-widths.":[33],"While":[34],"this":[35,49],"problem":[36],"has":[37],"been":[38],"observed":[39],"in":[40,111],"prior":[41],"work,":[42],"precise":[44],"cause":[45],"remains":[46],"unclear.":[47],"In":[48],"paper,":[50],"we":[51,122],"analyze":[52],"of":[55,61,81,95,109,141,161],"QAT":[56,174],"estimating":[58],"spectrum":[60,114],"loss-surface":[63],"Hessians.":[64],"We":[65],"find":[66],"that":[67,171],"converge":[70],"flat":[72],"regions":[73],"around":[74,99],"saddle":[75],"points,":[76],"where":[77],"a":[78],"large":[79],"fraction":[80,94],"Hessian":[83,96,113],"eigenvalues":[84,97,110],"are":[85,202],"both":[86],"positive":[87],"negative.":[89],"During":[90],"training,":[91],"an":[92,124],"increasing":[93,154],"concentrates":[98],"zero,":[100],"whose":[101],"magnitude":[102,108],"decreases.":[103],"At":[104],"lower":[105],"bit-widths,":[106],"significantly":[116],"smaller.":[117],"To":[118],"mitigate":[119],"these":[120],"issues,":[121],"propose":[123],"algorithm":[125],"called":[126],"WinQ":[127,172,191],"accelerate":[129],"QAT,":[130],"which":[131],"involves:":[132],"(1)":[133],"periodically":[134],"resetting":[135],"linear":[139],"interpolation":[140],"weights,":[145],"reducing":[146],"distance":[148],"quantization":[151,182,195,211],"grid":[152],"eigenvalue":[155],"magnitude,":[156],"(2)":[158],"computing":[159],"noise-injected":[162],"regularize":[165],"Hessian.":[167],"Extensive":[168],"experiments":[169],"show":[170],"accelerates":[173],"up":[176,197],"4":[178],"times":[179],"across":[180,204],"various":[181],"methods":[183],"models.":[185],"Under":[186],"same":[188],"cost,":[190],"improves":[192],"state-of-the-art":[193],"sub-4-bit":[194],"8.8%.":[199],"These":[200],"results":[201],"consistent":[203],"16":[205],"settings":[206],"with":[207],"different":[208],"models,":[210],"methods,":[212],"bit":[214],"widths.":[215]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-20T00:00:00"}