{"id":"https://openalex.org/W4412888086","doi":"https://doi.org/10.18653/v1/2025.findings-acl.694","title":"Continual Quantization-Aware Pre-Training: When to transition from 16-bit to 1.58-bit pre-training for BitNet language models?","display_name":"Continual Quantization-Aware Pre-Training: When to transition from 16-bit to 1.58-bit pre-training for BitNet language models?","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412888086","doi":"https://doi.org/10.18653/v1/2025.findings-acl.694"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.694","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.694","pdf_url":"https://aclanthology.org/2025.findings-acl.694.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.694.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072161944","display_name":"Jacob Woge Nielsen","orcid":"https://orcid.org/0000-0002-5466-7869"},"institutions":[{"id":"https://openalex.org/I177969490","display_name":"University of Southern Denmark","ror":"https://ror.org/03yrrjy16","country_code":"DK","type":"education","lineage":["https://openalex.org/I177969490"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Jacob Nielsen","raw_affiliation_strings":["Department of Mathematics and Computer Science University of Southern Denmark"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Mathematics and Computer Science University of Southern Denmark","institution_ids":["https://openalex.org/I177969490"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104621238","display_name":"Peter Schneider-Kamp","orcid":null},"institutions":[{"id":"https://openalex.org/I177969490","display_name":"University of Southern Denmark","ror":"https://ror.org/03yrrjy16","country_code":"DK","type":"education","lineage":["https://openalex.org/I177969490"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Peter Schneider-Kamp","raw_affiliation_strings":["Department of Mathematics and Computer Science University of Southern Denmark"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Mathematics and Computer Science University of Southern Denmark","institution_ids":["https://openalex.org/I177969490"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5114656612","display_name":"Lukas Galke","orcid":null},"institutions":[{"id":"https://openalex.org/I177969490","display_name":"University of Southern Denmark","ror":"https://ror.org/03yrrjy16","country_code":"DK","type":"education","lineage":["https://openalex.org/I177969490"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Lukas Galke","raw_affiliation_strings":["Department of Mathematics and Computer Science University of Southern Denmark"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Mathematics and Computer Science University of Southern Denmark","institution_ids":["https://openalex.org/I177969490"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I177969490"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"13483","last_page":"13493"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9747999906539917,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9747999906539917,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9115999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9078999757766724,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bit","display_name":"Bit (key)","score":0.8029468059539795},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.7595019340515137},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.686492383480072},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6652672290802002},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4542865753173828},{"id":"https://openalex.org/keywords/transition","display_name":"Transition (genetics)","score":0.4503520727157593},{"id":"https://openalex.org/keywords/arithmetic","display_name":"Arithmetic","score":0.41814178228378296},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3505885601043701},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.22206652164459229},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.17143800854682922},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1411440074443817}],"concepts":[{"id":"https://openalex.org/C117011727","wikidata":"https://www.wikidata.org/wiki/Q1278488","display_name":"Bit (key)","level":2,"score":0.8029468059539795},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.7595019340515137},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.686492383480072},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6652672290802002},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4542865753173828},{"id":"https://openalex.org/C194232998","wikidata":"https://www.wikidata.org/wiki/Q1606712","display_name":"Transition (genetics)","level":3,"score":0.4503520727157593},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.41814178228378296},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3505885601043701},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.22206652164459229},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.17143800854682922},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1411440074443817},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.694","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.694","pdf_url":"https://aclanthology.org/2025.findings-acl.694.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},{"id":"pmh:oai:sdu.dk:openaire/908bbdd8-9326-46b4-a3a6-561dc4ebce13","is_oa":true,"landing_page_url":"https://portal.findresearcher.sdu.dk/da/publications/908bbdd8-9326-46b4-a3a6-561dc4ebce13","pdf_url":null,"source":{"id":"https://openalex.org/S4306400423","display_name":"University of Southern Denmark Research Portal (University of Southern Denmark)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I177969490","host_organization_name":"University of Southern Denmark","host_organization_lineage":["https://openalex.org/I177969490"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.694","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.694","pdf_url":"https://aclanthology.org/2025.findings-acl.694.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412888086.pdf","grobid_xml":"https://content.openalex.org/works/W4412888086.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4327546585","https://openalex.org/W2411923897","https://openalex.org/W4394546135","https://openalex.org/W4285347720","https://openalex.org/W4200259850","https://openalex.org/W2333831899","https://openalex.org/W230091440","https://openalex.org/W2484894494","https://openalex.org/W2367385042","https://openalex.org/W4412233755"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,78,110],"(LLMs)":[3],"require":[4],"immense":[5],"resources":[6],"for":[7,24,73],"training":[8,43,71,107],"and":[9,28,62,85,108,132],"inference.Quantization,":[10],"a":[11,21,70],"technique":[12],"that":[13,42,98,139,150],"reduces":[14],"the":[15,77,121,125,129,143],"precision":[16,84],"of":[17,123,145],"model":[18,55],"parameters,":[19],"offers":[20],"promising":[22],"solution":[23],"improving":[25],"LLM":[26],"efficiency":[27],"sustainability.While":[29],"post-training":[30],"quantization":[31,136],"methods":[32],"typically":[33],"achieve":[34],"4-8":[35],"bits":[36,47],"per":[37,48],"parameter,":[38],"recent":[39],"research":[40],"suggests":[41],"LLMs":[44],"with":[45,82],"1.58":[46],"weight":[49],"parameter":[50],"from":[51],"scratch":[52],"can":[53,153],"maintain":[54],"accuracy":[56],"while":[57],"greatly":[58],"reducing":[59],"memory":[60],"requirements":[61],"energy":[63],"consumption":[64],"at":[65,128],"inference":[66],"time.Here,":[67],"we":[68],"investigate":[69,120],"strategy":[72,101],"quantizationaware":[74],"pre-training,":[75],"where":[76],"are":[79],"first":[80],"trained":[81],"16-bit":[83,117],"then":[86],"transition":[87,130],"into":[88],"1.58-bit":[89,106],"quantization-aware":[90],"training.Our":[91],"results":[92],"on":[93],"11":[94],"downstream":[95],"tasks":[96],"show":[97],"this":[99],"16-to-1.58-bittraining":[100],"is":[102],"preferable":[103],"over":[104],"full":[105],"leaves":[109],"closer":[111],"to":[112],"those":[113],"which":[114],"have":[115],"undergone":[116],"training.We":[118],"further":[119,157],"effects":[122,152],"retaining":[124],"optimizer":[126],"state":[127],"point":[131],"gradually":[133],"phasing":[134],"in":[135],"strength":[137],"-finding":[138],"both":[140],"techniques":[141],"alleviate":[142],"magnitude":[144],"loss":[146],"spikes,":[147],"but":[148],"also":[149],"these":[151],"be":[154],"compensated":[155],"through":[156],"training.":[158]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-19T07:52:34.831488","created_date":"2025-10-10T00:00:00"}