{"id":"https://openalex.org/W7202078769","doi":"https://doi.org/10.3103/s0005105526700391","title":"Analysis of the Efficiency of Subword Tokenizers in a Low-Resource Linguistic Environment: Implementation Experience for the Tajik Language","display_name":"Analysis of the Efficiency of Subword Tokenizers in a Low-Resource Linguistic Environment: Implementation Experience for the Tajik Language","publication_year":2026,"publication_date":"2026-08-01","ids":{"openalex":"https://openalex.org/W7202078769","doi":"https://doi.org/10.3103/s0005105526700391"},"language":"en","primary_location":{"id":"doi:10.3103/s0005105526700391","doi":"https://doi.org/10.3103/s0005105526700391","is_oa":false,"landing_page_url":"https://doi.org/10.3103/s0005105526700391","pdf_url":null,"source":{"id":"https://openalex.org/S18350981","display_name":"Automatic Documentation and Mathematical Linguistics","issn_l":"0005-1055","issn":["0005-1055","1934-8371"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["dongbi-c","poland-20","russia-white-list-4"],"host_organization":"https://openalex.org/P4310320267","host_organization_name":"Pleiades Publishing","host_organization_lineage":["https://openalex.org/P4310320267","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Pleiades Publishing","Springer Nature"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Automatic Documentation and Mathematical Linguistics","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135642748","display_name":"Mullosharaf K. Arabov","orcid":"https://orcid.org/0000-0003-2525-1183","observed_orcids":["https://orcid.org/0000-0003-2525-1183"]},"institutions":[{"id":"https://openalex.org/I21203515","display_name":"Kazan Federal University","ror":"https://ror.org/05256ym39","country_code":"RU","type":"education","lineage":["https://openalex.org/I21203515"]}],"countries":["RU"],"is_corresponding":true,"raw_author_name":"M. K. Arabov","raw_affiliation_strings":["Kazan Federal University, Kazan, Russia"],"raw_orcid":"https://orcid.org/0000-0003-2525-1183","affiliations":[{"raw_affiliation_string":"Kazan Federal University, Kazan, Russia","institution_ids":["https://openalex.org/I21203515"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5134248168","display_name":"Svetlana Sergeevna Khaybullina","orcid":"https://orcid.org/0009-0003-8818-2370","observed_orcids":["https://orcid.org/0009-0003-8818-2370"]},"institutions":[{"id":"https://openalex.org/I21203515","display_name":"Kazan Federal University","ror":"https://ror.org/05256ym39","country_code":"RU","type":"education","lineage":["https://openalex.org/I21203515"]}],"countries":["RU"],"is_corresponding":true,"raw_author_name":"S. S. Khaybullina","raw_affiliation_strings":["Kazan Federal University, Kazan, Russia"],"raw_orcid":"https://orcid.org/0009-0003-8818-2370","affiliations":[{"raw_affiliation_string":"Kazan Federal University, Kazan, Russia","institution_ids":["https://openalex.org/I21203515"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5134248168","https://openalex.org/A5135642748"],"corresponding_institution_ids":["https://openalex.org/I21203515"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"60","issue":"S2","first_page":"S225","last_page":"S230"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.004699999932199717,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13260","display_name":"Linguistics and Language Studies","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/subword-tokenization","display_name":"subword tokenization","score":0.9950000047683716},{"id":"https://openalex.org/keywords/tajik-language","display_name":"Tajik language","score":0.9599999785423279},{"id":"https://openalex.org/keywords/low-resource-languages","display_name":"low-resource languages","score":0.9980000257492065},{"id":"https://openalex.org/keywords/morphological-complexity","display_name":"morphological complexity","score":0.6809999942779541},{"id":"https://openalex.org/keywords/bpe","display_name":"BPE","score":0.703000009059906},{"id":"https://openalex.org/keywords/wordpiece","display_name":"WordPiece","score":0.9990000128746033},{"id":"https://openalex.org/keywords/unigrams","display_name":"unigrams","score":0.8679999709129333},{"id":"https://openalex.org/keywords/n-gram-frequencies","display_name":"n-gram frequencies","score":0.9440000057220459},{"id":"https://openalex.org/keywords/corpus-construction","display_name":"corpus construction","score":0.4090000092983246},{"id":"https://openalex.org/keywords/language-model-training","display_name":"language model training","score":0.34700000286102295}],"concepts":[{"id":"https://openalex.org/C176982825","wikidata":"https://www.wikidata.org/wiki/Q835922","display_name":"Lexical analysis","level":2,"score":0.9083999991416931},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8029000163078308},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.736299991607666},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6370000243186951},{"id":"https://openalex.org/C98501671","wikidata":"https://www.wikidata.org/wiki/Q1948408","display_name":"Text segmentation","level":3,"score":0.5378999710083008},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.5231000185012817},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.4490000009536743},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4345000088214874},{"id":"https://openalex.org/C12426560","wikidata":"https://www.wikidata.org/wiki/Q189569","display_name":"Basis (linear algebra)","level":2,"score":0.40880000591278076},{"id":"https://openalex.org/C161831844","wikidata":"https://www.wikidata.org/wiki/Q2554325","display_name":"Lemmatisation","level":2,"score":0.3763999938964844},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3474000096321106},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.33239999413490295},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.3181999921798706},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.31540000438690186},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.2849999964237213},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.2574000060558319}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.3103/s0005105526700391","doi":"https://doi.org/10.3103/s0005105526700391","is_oa":false,"landing_page_url":"https://doi.org/10.3103/s0005105526700391","pdf_url":null,"source":{"id":"https://openalex.org/S18350981","display_name":"Automatic Documentation and Mathematical Linguistics","issn_l":"0005-1055","issn":["0005-1055","1934-8371"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["dongbi-c","poland-20","russia-white-list-4"],"host_organization":"https://openalex.org/P4310320267","host_organization_name":"Pleiades Publishing","host_organization_lineage":["https://openalex.org/P4310320267","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Pleiades Publishing","Springer Nature"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Automatic Documentation and Mathematical Linguistics","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":5,"referenced_works":["https://openalex.org/W1816313093","https://openalex.org/W2885185669","https://openalex.org/W2897105542","https://openalex.org/W2963341956","https://openalex.org/W4390228511"],"related_works":["https://openalex.org/W7155203046","https://openalex.org/W7213431164","https://openalex.org/W7128404066","https://openalex.org/W7135237425","https://openalex.org/W7119726829","https://openalex.org/W4386214719","https://openalex.org/W7213457951","https://openalex.org/W7140096295","https://openalex.org/W4395064864","https://openalex.org/W7165988237"],"abstract_inverted_index":{"Abstract":[0],"Contemporary":[1],"approaches":[2,173],"to":[3,11,120,146,153,174,218],"subword":[4,83,175],"text":[5,131],"tokenization":[6,92,134,182],"are":[7],"considered":[8],"in":[9,200,224],"application":[10],"the":[12,31,34,86,99,106,127,148,151,155,161,167,179,185,188,192,201,219,225],"low-resource":[13,214],"Tajik":[14,193,211],"language,":[15],"which":[16],"is":[17],"characterised":[18],"by":[19],"a":[20,36,58,75,121],"complex":[21],"morphological":[22,156,189],"structure":[23],"and":[24,42,47,55,72,79,96,102,110,157,169,177,206,212],"high":[25],"variability":[26],"of":[27,33,52,69,82,88,123,150,160,171,187,191,203,221],"word":[28],"forms.":[29],"In":[30],"course":[32],"study,":[35],"large-scale":[37],"heterogeneous":[38],"corpus":[39,66],"was":[40,67,116],"compiled":[41],"preprocessed,":[43],"comprising":[44],"99":[45],"books":[46],"134":[48],"497":[49],"textual":[50],"articles":[51],"various":[53,172],"genres":[54],"topics,":[56],"with":[57],"total":[59],"volume":[60],"exceeding":[61],"33":[62],"million":[63],"tokens.":[64],"The":[65,163,195],"cleaned":[68],"noise,":[70],"normalised,":[71],"used":[73,199],"as":[74,136,138],"basis":[76,87],"for":[77,210],"training":[78],"subsequent":[80],"testing":[81],"models.":[84],"On":[85],"this":[89],"corpus,":[90],"five":[91],"models":[93,152,205],"were":[94],"trained":[95],"analyzed,":[97],"implementing":[98],"BPE,":[100],"WordPiece,":[101],"Unigram":[103],"algorithms":[104],"using":[105],"Hugging":[107],"Face":[108],"Tokenizers":[109],"SentencePiece":[111],"libraries.":[112],"A":[113],"comparative":[114],"assessment":[115],"carried":[117],"out":[118],"according":[119],"number":[122],"key":[124],"indicators,":[125],"including":[126],"out-of-vocabulary":[128],"(OOV)":[129],"rate,":[130],"compression":[132],"ratio,":[133],"speed,":[135],"well":[137],"n-gram":[139],"distribution":[140],"characteristics":[141],"that":[142],"make":[143],"it":[144],"possible":[145],"evaluate":[147],"ability":[149],"reflect":[154],"structural":[158],"organization":[159],"language.":[162,194],"experimental":[164],"results":[165],"revealed":[166],"strengths":[168],"weaknesses":[170],"segmentation":[176],"identified":[178],"most":[180],"effective":[181],"strategies":[183],"under":[184],"conditions":[186],"complexity":[190],"findings":[196],"may":[197],"be":[198],"development":[202],"language":[204],"applied":[207],"NLP":[208],"tools":[209],"other":[213],"languages,":[215],"thus":[216],"contributing":[217],"expansion":[220],"their":[222],"presence":[223],"digital":[226],"environment.":[227]},"counts_by_year":[],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2026-08-10T00:00:00"}