{"id":"https://openalex.org/W4396913941","doi":"https://doi.org/10.1016/j.tcs.2026.116131","title":"Constructing a BPE tokenization DFA","display_name":"Constructing a BPE tokenization DFA","publication_year":2026,"publication_date":"2026-06-23","ids":{"openalex":"https://openalex.org/W4396913941","doi":"https://doi.org/10.1016/j.tcs.2026.116131"},"language":"en","primary_location":{"id":"doi:10.1016/j.tcs.2026.116131","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.tcs.2026.116131","pdf_url":null,"source":{"id":"https://openalex.org/S90727058","display_name":"Theoretical Computer Science","issn_l":"0304-3975","issn":["0304-3975","1879-2294"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Theoretical Computer Science","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.tcs.2026.116131","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028862129","display_name":"Martin Berglund","orcid":"https://orcid.org/0000-0002-3692-6994"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Martin Berglund","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-3692-6994","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098657757","display_name":"Willeke Martens","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Willeke Martens","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5000026394","display_name":"Brink van der Merwe","orcid":"https://orcid.org/0000-0001-5010-9934"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Brink van der Merwe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2880,"currency":"USD","value_usd":2880},"apc_paid":{"value":2880,"currency":"USD","value_usd":2880},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.00165837,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"1083","issue":null,"first_page":"116131","last_page":"116131"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.6480000019073486,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.6480000019073486,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/lexical-analysis","display_name":"Lexical analysis","score":0.7360659241676331},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.48961812257766724},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.21471846103668213}],"concepts":[{"id":"https://openalex.org/C176982825","wikidata":"https://www.wikidata.org/wiki/Q835922","display_name":"Lexical analysis","level":2,"score":0.7360659241676331},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48961812257766724},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.21471846103668213}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1016/j.tcs.2026.116131","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.tcs.2026.116131","pdf_url":null,"source":{"id":"https://openalex.org/S90727058","display_name":"Theoretical Computer Science","issn_l":"0304-3975","issn":["0304-3975","1879-2294"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Theoretical Computer Science","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2405.07671","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.07671","pdf_url":"https://arxiv.org/pdf/2405.07671","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2405.07671","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2405.07671","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1016/j.tcs.2026.116131","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.tcs.2026.116131","pdf_url":null,"source":{"id":"https://openalex.org/S90727058","display_name":"Theoretical Computer Science","issn_l":"0304-3975","issn":["0304-3975","1879-2294"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Theoretical Computer Science","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322327","display_name":"Knut och Alice Wallenbergs Stiftelse","ror":"https://ror.org/004hzzk67"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W4386014872","https://openalex.org/W1847536016","https://openalex.org/W4361193986","https://openalex.org/W3149094754","https://openalex.org/W2148703997","https://openalex.org/W4366851046","https://openalex.org/W3172681236","https://openalex.org/W2033371749"],"abstract_inverted_index":{"Many":[0],"natural":[1],"language":[2],"processing":[3],"systems":[4],"operate":[5,35],"over":[6],"tokenizations":[7,38],"of":[8,28,69,85,100],"text":[9],"to":[10,34,52,59,92,109],"address":[11],"the":[12,25,41,60,86,97,101,119],"open-vocabulary":[13],"problem.":[14],"In":[15],"this":[16,91],"paper,":[17],"we":[18,89,106],"give":[19],"and":[20,57,72,88,123],"analyze":[21],"an":[22,111],"algorithm":[23],"for":[24],"efficient":[26],"construction":[27,80],"deterministic":[29],"finite":[30],"automata":[31,102],"(DFA)":[32],"designed":[33],"directly":[36],"on":[37,96],"produced":[39],"by":[40],"popular":[42],"byte":[43],"pair":[44],"encoding":[45],"(BPE)":[46],"technique.":[47],"This":[48],"makes":[49],"it":[50],"possible":[51],"apply":[53],"many":[54],"existing":[55],"techniques":[56],"algorithms":[58],"tokenized":[61,74],"case,":[62],"such":[63],"as":[64],"pattern":[65],"matching,":[66],"equivalence":[67],"checking":[68],"tokenization":[70],"dictionaries,":[71],"composing":[73],"languages":[75],"in":[76],"various":[77],"ways.":[78],"The":[79],"preserves":[81],"some":[82],"key":[83],"properties":[84],"automaton,":[87],"use":[90],"establish":[93],"asymptotic":[94],"bounds":[95],"state":[98],"complexity":[99],"that":[103],"result.":[104],"Finally,":[105],"demonstrate":[107],"how":[108],"construct":[110],"input-deterministic":[112],"(subsequential)":[113],"string-to-string":[114],"transducer":[115],"which":[116],"precisely":[117],"describes":[118],"relationship":[120],"between":[121],"strings":[122],"their":[124],"correct":[125],"tokenizations.":[126]},"counts_by_year":[],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}