{"id":"https://openalex.org/W7171561599","doi":"https://doi.org/10.48550/arxiv.2607.23362","title":"Joint Optimization for Greedy Longest-match Tokenization","display_name":"Joint Optimization for Greedy Longest-match Tokenization","publication_year":2026,"publication_date":"2026-07-25","ids":{"openalex":"https://openalex.org/W7171561599","doi":"https://doi.org/10.48550/arxiv.2607.23362"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.23362","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.23362","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.23362","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5143798292","display_name":"Adhiraj Singh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Singh, Adhiraj","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039227948","display_name":"Deepanshu Mody","orcid":"https://orcid.org/0000-0002-9483-3227"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mody, Deepanshu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5143749374","display_name":"Ghina Al Shdaifat","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shdaifat, Ghina Al","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119758176","display_name":"Hamza Alshamy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alshamy, Hamza","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012580224","display_name":"Adam Wiemerslage","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wiemerslage, Adam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5143800696","display_name":"Varshini Reddy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Reddy, Varshini","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5067117608","display_name":"Craig W. Schmidt","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schmidt, Craig W.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.8880000114440918,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.8880000114440918,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.05350000038743019,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.022199999541044235,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/greedy-algorithm","display_name":"Greedy algorithm","score":0.60589998960495},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5376999974250793},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.43619999289512634},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.42980000376701355},{"id":"https://openalex.org/keywords/data-compression","display_name":"Data compression","score":0.41990000009536743},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.36559998989105225},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.3596000075340271},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.3386000096797943}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7013000249862671},{"id":"https://openalex.org/C51823790","wikidata":"https://www.wikidata.org/wiki/Q504353","display_name":"Greedy algorithm","level":2,"score":0.60589998960495},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5376999974250793},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46810001134872437},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.43619999289512634},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.42980000376701355},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.41990000009536743},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.36559998989105225},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.3596000075340271},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.35339999198913574},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.3386000096797943},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33809998631477356},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.3328000009059906},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.3240000009536743},{"id":"https://openalex.org/C56086750","wikidata":"https://www.wikidata.org/wiki/Q6042592","display_name":"Integer programming","level":2,"score":0.3206999897956848},{"id":"https://openalex.org/C81845259","wikidata":"https://www.wikidata.org/wiki/Q290117","display_name":"Quadratic programming","level":2,"score":0.30970001220703125},{"id":"https://openalex.org/C193319292","wikidata":"https://www.wikidata.org/wiki/Q272172","display_name":"Hamming distance","level":2,"score":0.3050999939441681},{"id":"https://openalex.org/C311688","wikidata":"https://www.wikidata.org/wiki/Q2393193","display_name":"Time complexity","level":2,"score":0.30379998683929443},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.30070000886917114},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.29739999771118164},{"id":"https://openalex.org/C41045048","wikidata":"https://www.wikidata.org/wiki/Q202843","display_name":"Linear programming","level":2,"score":0.27649998664855957},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.27559998631477356},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.265500009059906},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.25099998712539673}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.23362","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.23362","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.23362","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.23362","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"work":[1],"has":[2],"shown":[3],"that":[4,73,142,208],"subword":[5],"vocabularies":[6],"can":[7,212],"be":[8],"trained":[9],"to":[10,34,189],"optimize":[11],"compression":[12,155,218],"for":[13,52,112],"a":[14,102,225],"specific":[15],"inference":[16,44],"rule":[17,45],"rather":[18],"than":[19,194],"relying":[20],"on":[21,134],"greedy":[22,35,157],"heuristics":[23],"such":[24],"as":[25,61,200],"Byte":[26],"Pair":[27],"Encoding":[28],"(BPE).":[29],"We":[30,48],"extend":[31],"this":[32],"approach":[33],"left-to-right":[36],"longest-match":[37,83,158],"decoding,":[38,159],"the":[39,79,86,90,98,130,135,152,168,201,216],"fast":[40],"and":[41,67,106,179,184],"widely":[42],"used":[43],"underlying":[46],"WordPiece.":[47],"introduce":[49,108],"Joint":[50],"Optimization":[51],"Greedy":[53],"Longest-Match":[54],"Tokenization":[55],"(JOLT),":[56],"which":[57],"formulates":[58],"vocabulary":[59,180,210],"learning":[60],"an":[62],"integer":[63],"program":[64],"over":[65],"vocabulary-selection":[66],"segmentation-choice":[68],"variables.":[69],"Greedy-consistency":[70],"constraints":[71],"ensure":[72],"each":[74],"optimized":[75],"segmentation":[76,80],"exactly":[77],"matches":[78],"produced":[81],"by":[82,221],"decoding":[84],"under":[85,156],"selected":[87],"vocabulary,":[88],"aligning":[89],"training":[91,136,177,202],"objective":[92],"with":[93,196],"deployment-time":[94],"tokenization.":[95],"To":[96],"scale":[97],"optimization,":[99],"we":[100],"solve":[101],"linear":[103],"programming":[104],"relaxation":[105,117],"selectively":[107],"higher-order":[109],"segmentations":[110],"only":[111],"unresolved":[113],"pretokens.":[114],"The":[115,138],"resulting":[116],"is":[118,144],"nearly":[119],"integral:":[120],"rounded":[121],"solutions":[122],"fall":[123],"within":[124,146],"0.008":[125],"-":[126,148,164],"0.176":[127],"%":[128,150,166,191],"of":[129,151,167,182,215,227],"LP":[131],"lower":[132],"bound":[133,139],"scope.":[137],"also":[140],"shows":[141],"BPE":[143,222],"already":[145],"1":[147],"2":[149],"best":[153],"achievable":[154],"while":[160,223],"JOLT":[161,186],"closes":[162],"89.6":[163],"99.4":[165],"remaining":[169],"gap.":[170],"On":[171],"held-out":[172],"validation":[173],"data":[174],"across":[175],"four":[176],"scopes":[178],"sizes":[181],"32,000":[183],"64,000,":[185],"produces":[187],"up":[188],"0.78":[190],"fewer":[192],"tokens":[193],"BPE,":[195],"improvements":[197],"generally":[198],"increasing":[199],"scope":[203],"grows.":[204],"These":[205],"results":[206],"demonstrate":[207],"inference-aligned":[209],"optimization":[211],"recover":[213],"most":[214],"limited":[217],"headroom":[219],"left":[220],"providing":[224],"certificate":[226],"near-optimality.":[228]},"counts_by_year":[],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2026-07-29T00:00:00"}