{"id":"https://openalex.org/W7165629635","doi":"https://doi.org/10.48550/arxiv.2606.23566","title":"LangMAP: A Language-Adaptive Approach to Tokenization","display_name":"LangMAP: A Language-Adaptive Approach to Tokenization","publication_year":2026,"publication_date":"2026-06-22","ids":{"openalex":"https://openalex.org/W7165629635","doi":"https://doi.org/10.48550/arxiv.2606.23566"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.23566","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.23566","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.23566","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139192844","display_name":"Clara Meister","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Meister, Clara","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114634016","display_name":"Suchir Salhan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Salhan, Suchir","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139204729","display_name":"Andrzej Szablewski","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Szablewski, Andrzej","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139149555","display_name":"Pietro Lesci","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lesci, Pietro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138675282","display_name":"Paula Buttery","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Buttery, Paula","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5102825308","display_name":"Tiago Pimentel","orcid":"https://orcid.org/0000-0002-5159-4641"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pimentel, Tiago","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.660099983215332,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.660099983215332,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.14800000190734863,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.02290000021457672,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/lexical-analysis","display_name":"Lexical analysis","score":0.9452999830245972},{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.5059000253677368},{"id":"https://openalex.org/keywords/lemmatisation","display_name":"Lemmatisation","score":0.4869000017642975},{"id":"https://openalex.org/keywords/syntax","display_name":"Syntax","score":0.4787999987602234},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4487999975681305},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.44830000400543213},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4140999913215637},{"id":"https://openalex.org/keywords/abstract-syntax-tree","display_name":"Abstract syntax tree","score":0.4120999872684479}],"concepts":[{"id":"https://openalex.org/C176982825","wikidata":"https://www.wikidata.org/wiki/Q835922","display_name":"Lexical analysis","level":2,"score":0.9452999830245972},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8652999997138977},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6643000245094299},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6606000065803528},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.5059000253677368},{"id":"https://openalex.org/C161831844","wikidata":"https://www.wikidata.org/wiki/Q2554325","display_name":"Lemmatisation","level":2,"score":0.4869000017642975},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.4787999987602234},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4487999975681305},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.44830000400543213},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4140999913215637},{"id":"https://openalex.org/C58646249","wikidata":"https://www.wikidata.org/wiki/Q127380","display_name":"Abstract syntax tree","level":3,"score":0.4120999872684479},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3774999976158142},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.36239999532699585},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.34950000047683716},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.32100000977516174},{"id":"https://openalex.org/C126706616","wikidata":"https://www.wikidata.org/wiki/Q2944660","display_name":"Lexical item","level":2,"score":0.31869998574256897},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.30640000104904175},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2888999879360199},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.26829999685287476},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.2572000026702881}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.23566","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.23566","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.23566","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.23566","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6895375847816467,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Language-specific":[0],"tokenizers":[1],"improve":[2],"tokenization":[3,52,65,118],"quality":[4],"and":[5,134],"the":[6,33,56,60,110,124,170],"downstream":[7],"performance":[8],"of":[9,35,109,123],"models":[10],"on":[11,169,178],"those":[12],"languages.":[13],"However,":[14],"using":[15],"such":[16],"a":[17,21,24,47,51,67,78,87,106],"tokenizer":[18,90],"comes":[19],"at":[20,103,119],"cost:":[22],"either":[23],"new":[25],"model":[26,39,81],"must":[27,40],"be":[28,41,74],"trained":[29],"from":[30,66,82],"scratch,":[31],"or":[32,84],"vocabulary":[34],"an":[36],"existing":[37],"pretrained":[38,88],"adapted.":[42],"We":[43],"propose":[44],"Language-adaptive":[45],"Maximum":[46],"Posteriori":[48],"(LangMAP)":[49],"Tokenization,":[50],"scheme":[53],"that":[54,113],"extends":[55],"UnigramLM":[57],"algorithm":[58,111],"to":[59,85,91],"multilingual":[61,79],"setting,":[62],"producing":[63],"language-specific":[64,117],"single":[68],"shared":[69],"vocabulary.":[70,97],"Notably,":[71],"LangMAP":[72,138,163],"can":[73],"used":[75],"when":[76],"training":[77,104],"language":[80,99],"scratch":[83],"adapt":[86],"model's":[89],"individual":[92],"languages":[93,147,171],"without":[94,121],"changing":[95],"its":[96,173],"While":[98],"labels":[100],"are":[101,161,175],"required":[102],"time,":[105],"key":[107],"feature":[108],"is":[112],"it":[114],"then":[115],"performs":[116],"inference":[120],"knowledge":[122],"input's":[125],"language.":[126],"Across":[127],"14":[128],"open-source":[129],"tokenizers,":[130],"9":[131,135],"natural":[132],"languages,":[133,137],"programming":[136],"improves":[139,164],"morphological":[140],"boundary":[141],"alignment":[142,149],"and,":[143],"for":[144],"all":[145],"coding":[146],"tested,":[148],"with":[150],"abstract":[151],"syntax":[152],"tree":[153],"(AST)":[154],"leaf":[155],"boundaries.":[156],"In":[157],"fine-tuning":[158],"experiments,":[159],"results":[160],"mixed:":[162],"target-language":[165],"grammatical":[166],"acceptability":[167],"(MultiBLiMP)":[168],"tested;":[172],"benefits":[174],"less":[176],"consistent":[177],"knowledge-related":[179],"tasks":[180],"(Global-PIQA,":[181],"Belebele).":[182]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-24T00:00:00"}