{"id":"https://openalex.org/W7119483144","doi":"https://doi.org/10.48550/arxiv.2601.03368","title":"A path to natural language through tokenisation and transformers","display_name":"A path to natural language through tokenisation and transformers","publication_year":2026,"publication_date":"2026-01-06","ids":{"openalex":"https://openalex.org/W7119483144","doi":"https://doi.org/10.48550/arxiv.2601.03368"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2601.03368","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2601.03368","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2601.03368","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009524117","display_name":"David S. Berman","orcid":"https://orcid.org/0000-0002-5382-1668"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Berman, David S.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5091983021","display_name":"Alexander G. Stapleton","orcid":"https://orcid.org/0009-0009-6784-7779"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stapleton, Alexander G.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.6355000138282776,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.6355000138282776,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12380","display_name":"Authorship Attribution and Profiling","score":0.121799997985363,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10849999636411667,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.7831000089645386},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.619700014591217},{"id":"https://openalex.org/keywords/zipfs-law","display_name":"Zipf's law","score":0.6189000010490417},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5634999871253967},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.5189999938011169},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4424999952316284},{"id":"https://openalex.org/keywords/information-theory","display_name":"Information theory","score":0.4088999927043915},{"id":"https://openalex.org/keywords/probability-distribution","display_name":"Probability distribution","score":0.3743000030517578},{"id":"https://openalex.org/keywords/novelty","display_name":"Novelty","score":0.3628000020980835}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.7831000089645386},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.701200008392334},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.619700014591217},{"id":"https://openalex.org/C125932096","wikidata":"https://www.wikidata.org/wiki/Q205472","display_name":"Zipf's law","level":2,"score":0.6189000010490417},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5634999871253967},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5189999938011169},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4424999952316284},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41499999165534973},{"id":"https://openalex.org/C52622258","wikidata":"https://www.wikidata.org/wiki/Q131222","display_name":"Information theory","level":2,"score":0.4088999927043915},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.3743000030517578},{"id":"https://openalex.org/C2778738651","wikidata":"https://www.wikidata.org/wiki/Q16546687","display_name":"Novelty","level":2,"score":0.3628000020980835},{"id":"https://openalex.org/C2778136018","wikidata":"https://www.wikidata.org/wiki/Q10350689","display_name":"Predictive power","level":2,"score":0.362199991941452},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34360000491142273},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.3409000039100647},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.3319999873638153},{"id":"https://openalex.org/C2986587452","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical analysis","level":2,"score":0.3248000144958496},{"id":"https://openalex.org/C2993724205","wikidata":"https://www.wikidata.org/wiki/Q315","display_name":"Human language","level":2,"score":0.322299987077713},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3070000112056732},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.302700012922287},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.3003999888896942},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.29499998688697815},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.2863999903202057},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.275299996137619},{"id":"https://openalex.org/C2987812609","wikidata":"https://www.wikidata.org/wiki/Q194158","display_name":"Bit rate","level":2,"score":0.25529998540878296},{"id":"https://openalex.org/C87868495","wikidata":"https://www.wikidata.org/wiki/Q750843","display_name":"Information processing","level":2,"score":0.2524000108242035},{"id":"https://openalex.org/C149717495","wikidata":"https://www.wikidata.org/wiki/Q117806","display_name":"Gamma distribution","level":2,"score":0.2517000138759613},{"id":"https://openalex.org/C2988416141","wikidata":"https://www.wikidata.org/wiki/Q6031139","display_name":"Information loss","level":2,"score":0.2500999867916107}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2601.03368","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2601.03368","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2601.03368","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2601.03368","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.5149394273757935}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Natural":[0],"languages":[1],"exhibit":[2],"striking":[3],"regularities":[4],"in":[5,34,105],"their":[6],"statistical":[7,187],"structure,":[8],"including":[9],"notably":[10],"the":[11,29,43,49,56,69,109,133,143,159,164],"emergence":[12],"of":[13,52,58,89,111,194],"Zipf's":[14],"and":[15,63],"Heaps'":[16],"laws.":[17],"Despite":[18],"this,":[19],"it":[20],"remains":[21],"broadly":[22],"unclear":[23],"how":[24,78,174],"these":[25,171],"properties":[26,193],"relate":[27],"to":[28,113,163],"modern":[30],"tokenisation":[31,153],"schemes":[32],"used":[33],"contemporary":[35],"transformer":[36],"models.":[37],"In":[38],"this":[39],"note,":[40],"we":[41,120],"analyse":[42],"information":[44],"content":[45],"(as":[46],"measured":[47],"by":[48],"Shannon":[50],"entropy)":[51],"various":[53],"corpora":[54,125],"under":[55],"assumption":[57],"a":[59,65,95,101,180,186],"Zipfian":[60,96],"frequency":[61],"distribution,":[62],"derive":[64],"closed-form":[66],"expression":[67],"for":[68],"slot":[70],"entropy":[71],"expectation":[72],"value.":[73],"We":[74],"then":[75],"empirically":[76],"investigate":[77],"byte--pair":[79],"encoding":[80],"(BPE)":[81],"transforms":[82],"corpus":[83],"statistics,":[84],"showing":[85],"that":[86,132,151,189],"recursive":[87],"applications":[88],"BPE":[90,129,144,175],"drive":[91],"token":[92,117,156],"frequencies":[93],"toward":[94],"power":[97],"law":[98],"while":[99],"inducing":[100],"characteristic":[102],"growth":[103],"pattern":[104],"empirical":[106,160],"entropy.":[107],"Utilizing":[108],"ability":[110],"transformers":[112],"learn":[114],"context":[115],"dependent":[116,166],"probability":[118],"distributions,":[119],"train":[121],"language":[122],"models":[123],"on":[124],"tokenised":[126],"at":[127],"varying":[128],"depths,":[130],"revealing":[131],"model":[134],"predictive":[135],"entropies":[136],"increasingly":[137],"agree":[138],"with":[139],"Zipf-derived":[140],"predictions":[141],"as":[142,179,185],"depth":[145],"increases.":[146],"Attention-based":[147],"diagnostics":[148],"further":[149],"indicate":[150],"deeper":[152],"reduces":[154],"local":[155],"dependencies,":[157],"bringing":[158],"distribution":[161],"closer":[162],"weakly":[165],"(near":[167],"IID)":[168],"regime.":[169],"Together,":[170],"results":[172],"clarify":[173],"acts":[176],"not":[177],"only":[178],"compression":[181],"mechanism":[182],"but":[183],"also":[184],"transform":[188],"reconstructs":[190],"key":[191],"informational":[192],"natural":[195],"language.":[196]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-01-09T00:00:00"}