{"id":"https://openalex.org/W4416778185","doi":"https://doi.org/10.48550/arxiv.2511.20849","title":"Length-MAX Tokenizer for Language Models","display_name":"Length-MAX Tokenizer for Language Models","publication_year":2025,"publication_date":"2025-11-25","ids":{"openalex":"https://openalex.org/W4416778185","doi":"https://doi.org/10.48550/arxiv.2511.20849"},"language":null,"primary_location":{"id":"pmh:oai:arXiv.org:2511.20849","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2511.20849","pdf_url":"https://arxiv.org/pdf/2511.20849","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2511.20849","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100318622","display_name":"Dong Dong","orcid":"https://orcid.org/0000-0001-9784-6472"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Dong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5080575294","display_name":"Weijie Su","orcid":"https://orcid.org/0000-0003-1787-1219"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Su, Weijie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2371000051498413,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2371000051498413,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.22630000114440918,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.06360000371932983,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.7803999781608582},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.7253000140190125},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.7087000012397766},{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.6783000230789185},{"id":"https://openalex.org/keywords/downstream","display_name":"Downstream (manufacturing)","score":0.6363999843597412},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.51910001039505},{"id":"https://openalex.org/keywords/maximization","display_name":"Maximization","score":0.5011000037193298},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.3626999855041504}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8102999925613403},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.7803999781608582},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.7253000140190125},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.7087000012397766},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.6783000230789185},{"id":"https://openalex.org/C2776207758","wikidata":"https://www.wikidata.org/wiki/Q5303302","display_name":"Downstream (manufacturing)","level":2,"score":0.6363999843597412},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.51910001039505},{"id":"https://openalex.org/C2776330181","wikidata":"https://www.wikidata.org/wiki/Q18358244","display_name":"Maximization","level":2,"score":0.5011000037193298},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4072999954223633},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3813999891281128},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.3626999855041504},{"id":"https://openalex.org/C43364308","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Byte","level":2,"score":0.33570000529289246},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.33390000462532043},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.32499998807907104},{"id":"https://openalex.org/C51823790","wikidata":"https://www.wikidata.org/wiki/Q504353","display_name":"Greedy algorithm","level":2,"score":0.3174000084400177},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.3172999918460846},{"id":"https://openalex.org/C2778348673","wikidata":"https://www.wikidata.org/wiki/Q739302","display_name":"Production (economics)","level":2,"score":0.30979999899864197},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.28299999237060547},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.27730000019073486},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.2768999934196472},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2533999979496002}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2511.20849","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2511.20849","pdf_url":"https://arxiv.org/pdf/2511.20849","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2511.20849","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2511.20849","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2511.20849","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2511.20849","pdf_url":"https://arxiv.org/pdf/2511.20849","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0],"introduce":[1],"a":[2,48,53,59,120,134],"new":[3],"tokenizer":[4,161,209],"for":[5,182],"language":[6,197],"models":[7,97],"that":[8,180],"minimizes":[9],"the":[10,17,40,86,91,159,167],"average":[11,183],"tokens":[12,20,72],"per":[13],"character,":[14],"thereby":[15],"reducing":[16,147],"number":[18],"of":[19],"needed":[21],"to":[22,28,38,83,118,194],"represent":[23],"text":[24,30],"during":[25,31],"training":[26],"and":[27,57,65,85,101,113,124,127,152,166,202,215,218],"generate":[29],"inference.":[32,224],"Our":[33],"method,":[34],"which":[35],"we":[36],"refer":[37],"as":[39,52],"Length-MAX":[41,160],"tokenizer,":[42],"obtains":[43],"its":[44],"vocabulary":[45,79,164],"by":[46,150,156,221],"casting":[47],"length-weighted":[49],"objective":[50],"maximization":[51],"graph":[54],"partitioning":[55],"problem":[56],"developing":[58],"greedy":[60],"approximation":[61],"algorithm.":[62],"On":[63],"FineWeb":[64],"diverse":[66],"domains,":[67],"it":[68],"yields":[69],"14--18\\%":[70],"fewer":[71,115],"than":[73,187],"Byte":[74],"Pair":[75],"Encoding":[76],"(BPE)":[77],"across":[78],"sizes":[80],"from":[81,104],"10K":[82],"50K,":[84],"reduction":[87],"is":[88,93,210],"13.0\\%":[89],"when":[90],"size":[92],"64K.":[94],"Training":[95],"GPT-2":[96],"at":[98,138,172,223],"124M,":[99,139],"355M,":[100],"1.3B":[102],"parameters":[103],"scratch":[105],"with":[106,133,212],"five":[107],"runs":[108],"each":[109],"shows":[110],"18.5\\%,":[111],"17.2\\%,":[112],"18.5\\%":[114],"steps,":[116],"respectively,":[117],"reach":[119],"fixed":[121],"validation":[122],"loss,":[123],"13.7\\%,":[125],"12.7\\%,":[126],"13.7\\%":[128],"lower":[129],"inference":[130],"latency,":[131],"together":[132],"16\\%":[135],"throughput":[136],"gain":[137],"while":[140],"consistently":[141],"improving":[142,204],"on":[143,174],"downstream":[144,206],"tasks":[145],"including":[146],"LAMBADA":[148],"perplexity":[149],"11.7\\%":[151],"enhancing":[153],"HellaSwag":[154],"accuracy":[155],"4.3\\%.":[157],"Moreover,":[158],"achieves":[162],"99.62\\%":[163],"coverage":[165],"out-of-vocabulary":[168],"rate":[169],"remains":[170],"low":[171],"0.12\\%":[173],"test":[175],"sets.":[176],"These":[177],"results":[178],"demonstrate":[179],"optimizing":[181],"token":[184],"length,":[185],"rather":[186],"frequency":[188],"alone,":[189],"offers":[190],"an":[191],"effective":[192],"approach":[193],"more":[195],"efficient":[196],"modeling":[198],"without":[199],"sacrificing":[200],"--":[201,205],"often":[203],"performance.":[207],"The":[208],"compatible":[211],"production":[213],"systems":[214],"reduces":[216],"embedding":[217],"KV-cache":[219],"memory":[220],"18\\%":[222]},"counts_by_year":[],"updated_date":"2026-07-03T08:13:44.112507","created_date":"2025-11-28T00:00:00"}