{"id":"https://openalex.org/W2963932686","doi":"https://doi.org/10.18653/v1/p16-1186","title":"Strategies for Training Large Vocabulary Neural Language Models","display_name":"Strategies for Training Large Vocabulary Neural Language Models","publication_year":2016,"publication_date":"2016-01-01","ids":{"openalex":"https://openalex.org/W2963932686","doi":"https://doi.org/10.18653/v1/p16-1186","mag":"2963932686"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p16-1186","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p16-1186","pdf_url":"https://www.aclweb.org/anthology/P16-1186.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P16-1186.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100734874","display_name":"Wenlin Chen","orcid":"https://orcid.org/0000-0002-0939-0733"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wenlin Chen","raw_affiliation_strings":["Facebook, Menlo Park, CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA","institution_ids":["https://openalex.org/I4210099336","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065912572","display_name":"David Grangier","orcid":"https://orcid.org/0000-0002-8847-9532"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Grangier","raw_affiliation_strings":["Facebook, Menlo Park, CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA","institution_ids":["https://openalex.org/I4210099336","https://openalex.org/I4210114444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5083212922","display_name":"Michael Auli","orcid":"https://orcid.org/0000-0001-5974-4459"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Auli","raw_affiliation_strings":["Facebook, Menlo Park, CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, CA","institution_ids":["https://openalex.org/I4210099336","https://openalex.org/I4210114444"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":103,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1975","last_page":"1985"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8472216725349426},{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.8467333316802979},{"id":"https://openalex.org/keywords/normalization","display_name":"Normalization (sociology)","score":0.7157101035118103},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6138837337493896},{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.6026751399040222},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5413520336151123},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.4927985966205597},{"id":"https://openalex.org/keywords/popularity","display_name":"Popularity","score":0.4865933656692505},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.45626142621040344},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.42860519886016846},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.41434380412101746},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.41396433115005493},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.40446749329566956},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.13893398642539978},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.07910501956939697}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8472216725349426},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.8467333316802979},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.7157101035118103},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6138837337493896},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.6026751399040222},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5413520336151123},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4927985966205597},{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.4865933656692505},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.45626142621040344},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.42860519886016846},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.41434380412101746},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.41396433115005493},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40446749329566956},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.13893398642539978},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.07910501956939697},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/p16-1186","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p16-1186","pdf_url":"https://www.aclweb.org/anthology/P16-1186.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/p16-1186","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p16-1186","pdf_url":"https://www.aclweb.org/anthology/P16-1186.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.6499999761581421}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2963932686.pdf","grobid_xml":"https://content.openalex.org/works/W2963932686.grobid-xml"},"referenced_works_count":36,"referenced_works":["https://openalex.org/W36903255","https://openalex.org/W179875071","https://openalex.org/W932413789","https://openalex.org/W1499253590","https://openalex.org/W1518951372","https://openalex.org/W1520465330","https://openalex.org/W1573488949","https://openalex.org/W1575384945","https://openalex.org/W1614298861","https://openalex.org/W1632114991","https://openalex.org/W2095705004","https://openalex.org/W2100664567","https://openalex.org/W2100714283","https://openalex.org/W2120861206","https://openalex.org/W2121227244","https://openalex.org/W2131462252","https://openalex.org/W2133564696","https://openalex.org/W2134800885","https://openalex.org/W2152790380","https://openalex.org/W2152808281","https://openalex.org/W2171928131","https://openalex.org/W2180952760","https://openalex.org/W2185726469","https://openalex.org/W2250379827","https://openalex.org/W2250539671","https://openalex.org/W2251098065","https://openalex.org/W2251682575","https://openalex.org/W2259472270","https://openalex.org/W2295800168","https://openalex.org/W2611669587","https://openalex.org/W2914484425","https://openalex.org/W2951714314","https://openalex.org/W2963084471","https://openalex.org/W2964308564","https://openalex.org/W2998704965","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W3107204728","https://openalex.org/W4287591324","https://openalex.org/W4226420367","https://openalex.org/W2980176872","https://openalex.org/W4300326282","https://openalex.org/W2742395793","https://openalex.org/W2268150819","https://openalex.org/W4287755480","https://openalex.org/W2810018382","https://openalex.org/W3113607506"],"abstract_inverted_index":{"Training":[0],"neural":[1,24],"network":[2],"language":[3],"models":[4,14],"over":[5],"large":[6,30],"vocabularies":[7],"is":[8],"computationally":[9],"costly":[10],"compared":[11],"to":[12,26,48,80],"count-based":[13],"such":[15],"as":[16],"Kneser-Ney.":[17,81],"We":[18,44,62],"present":[19],"a":[20,50],"systematic":[21],"comparison":[22],"of":[23,53,60],"strategies":[25],"represent":[27],"and":[28,41,55,78],"train":[29],"vocabularies,":[31],"including":[32],"softmax,":[33,35],"hierarchical":[34],"target":[36],"sampling,":[37],"noise":[38],"contrastive":[39],"estimation":[40],"self":[42,46],"normalization.":[43],"extend":[45],"normalization":[47],"be":[49],"proper":[51],"estimator":[52],"likelihood":[54],"introduce":[56],"an":[57],"efficient":[58],"variant":[59],"softmax.":[61],"evaluate":[63],"each":[64],"method":[65],"on":[66,72],"three":[67],"popular":[68],"benchmarks,":[69],"examining":[70],"performance":[71],"rare":[73],"words,":[74],"the":[75],"speed/accuracy":[76],"trade-off":[77],"complementarity":[79]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":9},{"year":2020,"cited_by_count":12},{"year":2019,"cited_by_count":18},{"year":2018,"cited_by_count":23},{"year":2017,"cited_by_count":17},{"year":2016,"cited_by_count":9}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}