{"id":"https://openalex.org/W4394781992","doi":"https://doi.org/10.48550/arxiv.2404.07647","title":"Why do small language models underperform? Studying Language Model Saturation via the Softmax Bottleneck","display_name":"Why do small language models underperform? Studying Language Model Saturation via the Softmax Bottleneck","publication_year":2024,"publication_date":"2024-04-11","ids":{"openalex":"https://openalex.org/W4394781992","doi":"https://doi.org/10.48550/arxiv.2404.07647"},"language":"en","primary_location":{"id":"pmh:oai:HAL:hal-04546134v1","doi":null,"is_oa":true,"landing_page_url":"https://inria.hal.science/hal-04546134","pdf_url":"http://arxiv.org/pdf/2404.07647","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2024","raw_type":"info:eu-repo/semantics/preprint"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2404.07647","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071654594","display_name":"Nathan Godey","orcid":"https://orcid.org/0000-0001-8319-8128","observed_orcids":["https://orcid.org/0000-0001-8319-8128"]},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I39804081","display_name":"Sorbonne Universit\u00e9","ror":"https://ror.org/02en5vm52","country_code":"FR","type":"education","lineage":["https://openalex.org/I39804081"]},{"id":"https://openalex.org/I4405261730","display_name":"ALMANACH: Mod\u00e9lisation et analyse linguistique automatique et humanit\u00e9s computationnelles","ror":"https://ror.org/05rnwy495","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1326498283","https://openalex.org/I4405257605","https://openalex.org/I4405261730"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Godey, Nathan","raw_affiliation_strings":["Inria Paris ,","Sorbonne Universit\u00e9 Paris , France","ALMAnaCH - Automatic Language Modelling and ANAlysis & Computational Humanities (France)","SU - Sorbonne Universite\u0301 (21 rue de l\u2019\u00c9cole de m\u00e9decine - 75006 Paris - France)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Inria Paris ,","institution_ids":["https://openalex.org/I1326498283"]},{"raw_affiliation_string":"Sorbonne Universit\u00e9 Paris , France","institution_ids":["https://openalex.org/I39804081"]},{"raw_affiliation_string":"ALMAnaCH - Automatic Language Modelling and ANAlysis & Computational Humanities (France)","institution_ids":["https://openalex.org/I4405261730"]},{"raw_affiliation_string":"SU - Sorbonne Universite\u0301 (21 rue de l\u2019\u00c9cole de m\u00e9decine - 75006 Paris - France)","institution_ids":["https://openalex.org/I39804081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037164199","display_name":"\u00c9ric Villemonte de la Clergerie","orcid":"https://orcid.org/0000-0001-6428-9219","observed_orcids":["https://orcid.org/0000-0001-6428-9219"]},"institutions":[{"id":"https://openalex.org/I4405261730","display_name":"ALMANACH: Mod\u00e9lisation et analyse linguistique automatique et humanit\u00e9s computationnelles","ror":"https://ror.org/05rnwy495","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1326498283","https://openalex.org/I4405257605","https://openalex.org/I4405261730"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"de la Clergerie, \u00c9ric","raw_affiliation_strings":["ALMAnaCH - Automatic Language Modelling and ANAlysis & Computational Humanities (France)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ALMAnaCH - Automatic Language Modelling and ANAlysis & Computational Humanities (France)","institution_ids":["https://openalex.org/I4405261730"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077663332","display_name":"Beno\u00eet Sagot","orcid":"https://orcid.org/0000-0002-0107-8526","observed_orcids":["https://orcid.org/0000-0002-0107-8526"]},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I4405261730","display_name":"ALMANACH: Mod\u00e9lisation et analyse linguistique automatique et humanit\u00e9s computationnelles","ror":"https://ror.org/05rnwy495","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1326498283","https://openalex.org/I4405257605","https://openalex.org/I4405261730"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Sagot, Beno\u00eet","raw_affiliation_strings":["Inria Paris ,","ALMAnaCH - Automatic Language Modelling and ANAlysis & Computational Humanities (France)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Inria Paris ,","institution_ids":["https://openalex.org/I1326498283"]},{"raw_affiliation_string":"ALMAnaCH - Automatic Language Modelling and ANAlysis & Computational Humanities (France)","institution_ids":["https://openalex.org/I4405261730"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9496999979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9496999979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.03959999978542328,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.002300000051036477,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/small-language-models","display_name":"small language models","score":0.8489999771118164},{"id":"https://openalex.org/keywords/hidden-dimensions","display_name":"hidden dimensions","score":0.7310000061988831},{"id":"https://openalex.org/keywords/pretraining","display_name":"pre-training","score":0.593999981880188},{"id":"https://openalex.org/keywords/performance-plateau","display_name":"performance plateau","score":0.550000011920929}],"concepts":[{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.8979772329330444},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.8139266967773438},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7232246994972229},{"id":"https://openalex.org/C165464430","wikidata":"https://www.wikidata.org/wiki/Q1570441","display_name":"Parameterized complexity","level":2,"score":0.5514413714408875},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5315755605697632},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5133827924728394},{"id":"https://openalex.org/C9930424","wikidata":"https://www.wikidata.org/wiki/Q7426587","display_name":"Saturation (graph theory)","level":2,"score":0.5074021220207214},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4984276294708252},{"id":"https://openalex.org/C2777413886","wikidata":"https://www.wikidata.org/wiki/Q3276013","display_name":"Fluency","level":2,"score":0.4769419729709625},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4501892030239105},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4022733271121979},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3683549165725708},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.20669350028038025},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.13970991969108582},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.0},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:HAL:hal-04546134v1","doi":null,"is_oa":true,"landing_page_url":"https://inria.hal.science/hal-04546134","pdf_url":"http://arxiv.org/pdf/2404.07647","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2024","raw_type":"info:eu-repo/semantics/preprint"},{"id":"pmh:oai:arXiv.org:2404.07647","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.07647","pdf_url":"https://arxiv.org/pdf/2404.07647","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2404.07647","doi":"https://doi.org/10.48550/arxiv.2404.07647","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2404.07647","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:HAL:hal-04546134v1","doi":null,"is_oa":true,"landing_page_url":"https://inria.hal.science/hal-04546134","pdf_url":"http://arxiv.org/pdf/2404.07647","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2024","raw_type":"info:eu-repo/semantics/preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G3010527862","display_name":null,"funder_award_id":"ANR\u201019","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G5838787665","display_name":"PaRis Artificial Intelligence Research InstitutE","funder_award_id":"ANR-19-P3IA-0001","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"}],"funders":[{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"},{"id":"https://openalex.org/F4320326256","display_name":"Grand \u00c9quipement National De Calcul Intensif","ror":"https://ror.org/0010d1q40"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4394781992.pdf","grobid_xml":"https://content.openalex.org/works/W4394781992.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4404781877","https://openalex.org/W2767321762","https://openalex.org/W4385570327","https://openalex.org/W7148389315","https://openalex.org/W4312089714","https://openalex.org/W7117322780","https://openalex.org/W7135104214","https://openalex.org/W4416111891","https://openalex.org/W4307413505","https://openalex.org/W7161068304"],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2,6,27,52,58,106,123,143],"language":[3],"modeling":[4],"consist":[5],"pretraining":[7],"highly":[8],"parameterized":[9],"neural":[10],"networks":[11],"on":[12,131],"extremely":[13],"large":[14],"web-mined":[15],"text":[16],"corpora.":[17],"Training":[18],"and":[19,85,126],"inference":[20],"with":[21],"such":[22,70,107],"models":[23,43,84,108,129],"can":[24,44,72],"be":[25,73],"costly":[26],"practice,":[28],"which":[29,146],"incentivizes":[30],"the":[31,79,86,90,98,101,110,117,120],"use":[32],"of":[33,82,89,100,119],"smaller":[34,42,83],"counterparts.":[35],"However,":[36],"it":[37],"has":[38],"been":[39],"observed":[40],"that":[41,69,128],"suffer":[45],"from":[46],"saturation,":[47],"characterized":[48],"as":[49],"a":[50,62,76],"drop":[51],"performance":[53,99],"at":[54],"some":[55],"advanced":[56],"point":[57],"training":[59],"followed":[60],"by":[61,75],"plateau.":[63],"In":[64],"this":[65],"paper,":[66],"we":[67],"find":[68,127],"saturation":[71],"explained":[74],"mismatch":[77,96],"between":[78],"hidden":[80,135],"dimension":[81],"high":[87],"rank":[88],"target":[91],"contextual":[92],"probability":[93],"distribution.":[94],"This":[95],"affects":[97],"linear":[102],"prediction":[103],"head":[104],"used":[105],"through":[109],"well-known":[111],"softmax":[112,121],"bottleneck":[113,122],"phenomenon.":[114],"We":[115],"measure":[116],"effect":[118],"various":[124],"settings":[125],"based":[130],"less":[132],"than":[133],"1000":[134],"dimensions":[136],"tend":[137],"to":[138,148],"adopt":[139],"degenerate":[140],"latent":[141],"representations":[142],"late":[144],"pretraining,":[145],"leads":[147],"reduced":[149],"evaluation":[150],"performance.":[151]},"counts_by_year":[],"updated_date":"2026-10-25T09:29:13.331239","created_date":"2024-04-13T00:00:00"}