{"id":"https://openalex.org/W4416251585","doi":"https://doi.org/10.1109/ijcnn64981.2025.11228044","title":"Training Dynamics of a 1.7B LLaMa Model: A Data-Efficient Approach","display_name":"Training Dynamics of a 1.7B LLaMa Model: A Data-Efficient Approach","publication_year":2025,"publication_date":"2025-06-30","ids":{"openalex":"https://openalex.org/W4416251585","doi":"https://doi.org/10.1109/ijcnn64981.2025.11228044"},"language":null,"primary_location":{"id":"doi:10.1109/ijcnn64981.2025.11228044","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn64981.2025.11228044","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063351983","display_name":"Miles Q. Li","orcid":"https://orcid.org/0000-0001-7091-3268"},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Miles Q. Li","raw_affiliation_strings":["McGill University,School of Information Studies,Montreal,Canada"],"affiliations":[{"raw_affiliation_string":"McGill University,School of Information Studies,Montreal,Canada","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021788449","display_name":"Benjamin C. M. Fung","orcid":"https://orcid.org/0000-0001-8423-2906"},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Benjamin C. M. Fung","raw_affiliation_strings":["McGill University,School of Information Studies,Montreal,Canada"],"affiliations":[{"raw_affiliation_string":"McGill University,School of Information Studies,Montreal,Canada","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071168688","display_name":"Shih\u2010Chia Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I118292597","display_name":"National Taipei University of Technology","ror":"https://ror.org/00cn92c09","country_code":"TW","type":"education","lineage":["https://openalex.org/I118292597"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Shih-Chia Huang","raw_affiliation_strings":["National Taipei University of Technology,Department of Electronic Engineering,Taipei,Taiwan"],"affiliations":[{"raw_affiliation_string":"National Taipei University of Technology,Department of Electronic Engineering,Taipei,Taiwan","institution_ids":["https://openalex.org/I118292597"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5063351983"],"corresponding_institution_ids":["https://openalex.org/I5023651"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.18290941,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1550000011920929,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1550000011920929,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1395999938249588,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.07509999722242355,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.6510999798774719},{"id":"https://openalex.org/keywords/dynamics","display_name":"Dynamics (music)","score":0.49239999055862427},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.4717999994754791},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.439300000667572},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.41609999537467957},{"id":"https://openalex.org/keywords/qualitative-property","display_name":"Qualitative property","score":0.33820000290870667},{"id":"https://openalex.org/keywords/qualitative-research","display_name":"Qualitative research","score":0.33250001072883606},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.33239999413490295}],"concepts":[{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6510999798774719},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6363000273704529},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.49239999055862427},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.4717999994754791},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.439300000667572},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.41609999537467957},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3930000066757202},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3675000071525574},{"id":"https://openalex.org/C87156501","wikidata":"https://www.wikidata.org/wiki/Q7268708","display_name":"Qualitative property","level":2,"score":0.33820000290870667},{"id":"https://openalex.org/C190248442","wikidata":"https://www.wikidata.org/wiki/Q839486","display_name":"Qualitative research","level":2,"score":0.33250001072883606},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.33239999413490295},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3292999863624573},{"id":"https://openalex.org/C3018587665","wikidata":"https://www.wikidata.org/wiki/Q7268696","display_name":"Qualitative analysis","level":3,"score":0.32820001244544983},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32580000162124634},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.3093000054359436},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.3052000105381012},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.3052000105381012},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.27799999713897705},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2752000093460083},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2734000086784363},{"id":"https://openalex.org/C77405623","wikidata":"https://www.wikidata.org/wiki/Q598451","display_name":"System dynamics","level":2,"score":0.262800008058548},{"id":"https://openalex.org/C44280652","wikidata":"https://www.wikidata.org/wiki/Q104837","display_name":"Phase (matter)","level":2,"score":0.26170000433921814},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.25920000672340393}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ijcnn64981.2025.11228044","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn64981.2025.11228044","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320320994","display_name":"Canada Research Chairs","ror":"https://ror.org/0517h6h17"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":4,"referenced_works":["https://openalex.org/W2890894339","https://openalex.org/W2946609015","https://openalex.org/W2996908057","https://openalex.org/W2998617917"],"related_works":[],"abstract_inverted_index":{"Pretraining":[0],"large":[1],"language":[2],"models":[3],"is":[4],"a":[5,35,84],"complex":[6],"endeavor":[7],"influenced":[8],"by":[9],"multiple":[10],"factors,":[11],"including":[12],"model":[13,93,138],"architecture,":[14],"data":[15,150],"quality,":[16],"training":[17,33,55,126,160,168],"continuity,":[18],"and":[19,63,119,128,151,166,171,180],"hardware":[20,123],"constraints.":[21],"In":[22],"this":[23],"paper,":[24],"we":[25,78,174],"share":[26],"insights":[27],"gained":[28],"from":[29,68,117],"the":[30,53,92,109,120],"experience":[31],"of":[32,47,111,122,137],"DMaS-LLaMa-Lite,":[34],"fully":[36],"open":[37],"source,":[38],"1.7-billionparameter,":[39],"LLaMa-based":[40],"model,":[41],"on":[42,88,125],"approximately":[43],"20":[44],"billion":[45],"tokens":[46],"carefully":[48],"curated":[49],"data.":[50],"We":[51,103],"chronicle":[52],"full":[54],"trajectory,":[56],"documenting":[57],"how":[58,148],"evolving":[59],"validation":[60],"loss":[61],"levels":[62],"downstream":[64],"benchmarks":[65],"reflect":[66],"transitions":[67],"incoherent":[69],"text":[70],"to":[71,82,96,143,176],"fluent,":[72],"contextually":[73,99],"grounded":[74],"output.":[75],"Beyond":[76],"pretraining,":[77],"extend":[79],"our":[80,140],"analysis":[81,141],"include":[83],"post-training":[85],"phase":[86],"focused":[87],"instruction":[89],"tuning,":[90],"where":[91],"was":[94],"refined":[95],"produce":[97],"more":[98],"appropriate,":[100],"user-aligned":[101],"responses.":[102],"highlight":[104],"practical":[105],"considerations":[106],"such":[107],"as":[108],"importance":[110],"restoring":[112],"optimizer":[113],"states":[114],"when":[115],"resuming":[116],"checkpoints,":[118,170],"impact":[121],"changes":[124],"stability":[127],"throughput.":[129],"While":[130],"qualitative":[131],"evaluation":[132],"provides":[133],"an":[134],"intuitive":[135],"understanding":[136],"improvements,":[139],"extends":[142],"various":[144],"performance":[145],"benchmarks,":[146],"demonstrating":[147],"high-quality":[149],"thoughtful":[152],"scaling":[153],"enable":[154],"competitive":[155],"results":[156],"with":[157],"significantly":[158],"fewer":[159],"tokens.":[161],"By":[162],"detailing":[163],"these":[164],"experiences":[165],"offering":[167],"logs,":[169],"sample":[172],"outputs,":[173],"aim":[175],"guide":[177],"future":[178],"researchers":[179],"practitioners":[181],"in":[182],"refining":[183],"their":[184],"pretraining":[185],"strategies.":[186]},"counts_by_year":[],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-11-14T00:00:00"}