{"id":"https://openalex.org/W4409418368","doi":"https://doi.org/10.2139/ssrn.5217325","title":"Training Dynamics of a 1.7b Llama Model: A Data-Efficient Approach","display_name":"Training Dynamics of a 1.7b Llama Model: A Data-Efficient Approach","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4409418368","doi":"https://doi.org/10.2139/ssrn.5217325"},"language":"en","primary_location":{"id":"doi:10.2139/ssrn.5217325","doi":"https://doi.org/10.2139/ssrn.5217325","is_oa":true,"landing_page_url":"https://doi.org/10.2139/ssrn.5217325","pdf_url":null,"source":{"id":"https://openalex.org/S4210172589","display_name":"SSRN Electronic Journal","issn_l":"1556-5068","issn":["1556-5068"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1318003438","host_organization_name":"RELX Group (Netherlands)","host_organization_lineage":["https://openalex.org/I1318003438"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.2139/ssrn.5217325","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063351983","display_name":"Miles Q. Li","orcid":"https://orcid.org/0000-0001-7091-3268","observed_orcids":["https://orcid.org/0000-0001-7091-3268"]},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Miles Q. Li","raw_affiliation_strings":["McGill University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"McGill University","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021788449","display_name":"Benjamin C. M. Fung","orcid":"https://orcid.org/0000-0001-8423-2906","observed_orcids":["https://orcid.org/0000-0001-8423-2906"]},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Benjamin C. M. Fung","raw_affiliation_strings":["McGill University","McGill University, 3661 Peel Street, Montreal, Quebec H3A 1X1, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"McGill University","institution_ids":["https://openalex.org/I5023651"]},{"raw_affiliation_string":"McGill University, 3661 Peel Street, Montreal, Quebec H3A 1X1, Canada","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062271417","display_name":"Shih-Chia Huang","orcid":"https://orcid.org/0000-0002-6896-3415","observed_orcids":["https://orcid.org/0000-0002-6896-3415"]},"institutions":[{"id":"https://openalex.org/I118292597","display_name":"National Taipei University of Technology","ror":"https://ror.org/00cn92c09","country_code":"TW","type":"education","lineage":["https://openalex.org/I118292597"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Shih-Chia Huang","raw_affiliation_strings":["National Taipei University of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Taipei University of Technology","institution_ids":["https://openalex.org/I118292597"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5021788449"],"corresponding_institution_ids":["https://openalex.org/I5023651"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9423999786376953,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9423999786376953,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.04520000144839287,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0031999999191612005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6259999871253967},{"id":"https://openalex.org/keywords/training-dynamics","display_name":"training dynamics","score":0.6179999709129333},{"id":"https://openalex.org/keywords/data-efficiency","display_name":"data efficiency","score":0.6230000257492065},{"id":"https://openalex.org/keywords/instruction-tuning","display_name":"instruction tuning","score":0.8040000200271606},{"id":"https://openalex.org/keywords/checkpointing","display_name":"checkpointing","score":0.6019999980926514},{"id":"https://openalex.org/keywords/training-throughput","display_name":"training throughput","score":0.6399999856948853},{"id":"https://openalex.org/keywords/training-stability","display_name":"training stability","score":0.6340000033378601},{"id":"https://openalex.org/keywords/validation-loss","display_name":"validation loss","score":0.5090000033378601},{"id":"https://openalex.org/keywords/open-source-language-models","display_name":"open-source language models","score":0.5709999799728394},{"id":"https://openalex.org/keywords/model-scaling","display_name":"model scaling","score":0.4129999876022339}],"concepts":[{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6623439788818359},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.6519670486450195},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5791493654251099},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3371261954307556},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1768919825553894},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.11630657315254211},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.2139/ssrn.5217325","doi":"https://doi.org/10.2139/ssrn.5217325","is_oa":true,"landing_page_url":"https://doi.org/10.2139/ssrn.5217325","pdf_url":null,"source":{"id":"https://openalex.org/S4210172589","display_name":"SSRN Electronic Journal","issn_l":"1556-5068","issn":["1556-5068"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1318003438","host_organization_name":"RELX Group (Netherlands)","host_organization_lineage":["https://openalex.org/I1318003438"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.2139/ssrn.5217325","doi":"https://doi.org/10.2139/ssrn.5217325","is_oa":true,"landing_page_url":"https://doi.org/10.2139/ssrn.5217325","pdf_url":null,"source":{"id":"https://openalex.org/S4210172589","display_name":"SSRN Electronic Journal","issn_l":"1556-5068","issn":["1556-5068"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1318003438","host_organization_name":"RELX Group (Netherlands)","host_organization_lineage":["https://openalex.org/I1318003438"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4405783338","https://openalex.org/W4403965217","https://openalex.org/W7127050266","https://openalex.org/W4404307413","https://openalex.org/W7083814546","https://openalex.org/W4393905476","https://openalex.org/W4416415790","https://openalex.org/W4402427048","https://openalex.org/W7110811643","https://openalex.org/W4404392665"],"abstract_inverted_index":null,"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2025-04-15T00:00:00"}