{"id":"https://openalex.org/W4415798175","doi":"https://doi.org/10.52202/079017-0914","title":"Not All Tokens Are What You Need for Pretraining","display_name":"Not All Tokens Are What You Need for Pretraining","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4415798175","doi":"https://doi.org/10.52202/079017-0914"},"language":"en","primary_location":{"id":"doi:10.52202/079017-0914","doi":"https://doi.org/10.52202/079017-0914","is_oa":false,"landing_page_url":"https://doi.org/10.52202/079017-0914","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 37","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007367450","display_name":"Zhenghao Lin","orcid":"https://orcid.org/0009-0009-3051-193X","observed_orcids":["https://orcid.org/0009-0009-3051-193X","https://orcid.org/0000-0003-3926-6140"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhenghao Lin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zhibin Gou","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhibin Gou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041448669","display_name":"Yeyun Gong","orcid":"https://orcid.org/0000-0001-9954-9674","observed_orcids":["https://orcid.org/0000-0001-9954-9674"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yeyun Gong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Xiao Liu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101180037","display_name":"Yelong Shen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yelong Shen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ruochen Xu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ruochen Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100745599","display_name":"Lin Chen","orcid":"https://orcid.org/0000-0002-2290-1630","observed_orcids":["https://orcid.org/0000-0002-2290-1630","https://orcid.org/0000-0003-1397-6769"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen Lin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020953714","display_name":"Yujiu Yang","orcid":"https://orcid.org/0000-0002-6427-1024","observed_orcids":["https://orcid.org/0000-0002-6427-1024"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yujiu Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074199611","display_name":"Jian Jiao","orcid":"https://orcid.org/0000-0003-4779-9588","observed_orcids":["https://orcid.org/0000-0003-4779-9588"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jian Jiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Nan Duan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nan Duan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5051745436","display_name":"Weizhu Chen","orcid":"https://orcid.org/0000-0003-4196-7229","observed_orcids":["https://orcid.org/0000-0003-4196-7229"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Weizhu Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.2101,"has_fulltext":false,"cited_by_count":14,"citation_normalized_percentile":{"value":0.91274379,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"29029","last_page":"29063"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8432999849319458,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8432999849319458,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0689999982714653,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.017999999225139618,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pretraining","display_name":"pre-training","score":0.7149999737739563},{"id":"https://openalex.org/keywords/tokenization","display_name":"tokenization","score":0.6330000162124634},{"id":"https://openalex.org/keywords/token-selection","display_name":"token selection","score":0.41999998688697815},{"id":"https://openalex.org/keywords/language-models","display_name":"language models","score":0.5379999876022339},{"id":"https://openalex.org/keywords/representation-learning","display_name":"representation learning","score":0.35899999737739563}],"concepts":[],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.52202/079017-0914","doi":"https://doi.org/10.52202/079017-0914","is_oa":false,"landing_page_url":"https://doi.org/10.52202/079017-0914","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 37","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7196965237","https://openalex.org/W7196937537","https://openalex.org/W7197001364","https://openalex.org/W4415795638","https://openalex.org/W7133186241","https://openalex.org/W4415797788","https://openalex.org/W4415797039","https://openalex.org/W7196948905","https://openalex.org/W7196961488","https://openalex.org/W7197014472"],"abstract_inverted_index":null,"counts_by_year":[{"year":2026,"cited_by_count":14}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-11-03T00:00:00"}