{"id":"https://openalex.org/W7155095908","doi":"https://doi.org/10.48550/arxiv.2604.17105","title":"How Tokenization Limits Phonological Knowledge Representation in Language Models and How to Improve Them","display_name":"How Tokenization Limits Phonological Knowledge Representation in Language Models and How to Improve Them","publication_year":2026,"publication_date":"2026-04-18","ids":{"openalex":"https://openalex.org/W7155095908","doi":"https://doi.org/10.48550/arxiv.2604.17105"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.17105","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.17105","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.17105","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5132697773","display_name":"Disen Liao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liao, Disen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134137465","display_name":"Freda Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Freda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.1550000011920929,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.1550000011920929,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10465","display_name":"Neurobiology of Language and Bilingualism","score":0.13330000638961792,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10403","display_name":"Phonetics and Phonology Research","score":0.12200000137090683,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/lexical-analysis","display_name":"Lexical analysis","score":0.8421000242233276},{"id":"https://openalex.org/keywords/syllable","display_name":"Syllable","score":0.5475999712944031},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.4749000072479248},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.47029998898506165},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4636000096797943},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.44519999623298645},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.42890000343322754}],"concepts":[{"id":"https://openalex.org/C176982825","wikidata":"https://www.wikidata.org/wiki/Q835922","display_name":"Lexical analysis","level":2,"score":0.8421000242233276},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7670000195503235},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6265000104904175},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6055999994277954},{"id":"https://openalex.org/C109089402","wikidata":"https://www.wikidata.org/wiki/Q8188","display_name":"Syllable","level":2,"score":0.5475999712944031},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.4749000072479248},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.47029998898506165},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4636000096797943},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.44519999623298645},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.42890000343322754},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.41019999980926514},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.40130001306533813},{"id":"https://openalex.org/C2987487971","wikidata":"https://www.wikidata.org/wiki/Q8096","display_name":"Lexical access","level":3,"score":0.3596000075340271},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.2883000075817108},{"id":"https://openalex.org/C4768521","wikidata":"https://www.wikidata.org/wiki/Q512366","display_name":"Phonotactics","level":3,"score":0.2858999967575073},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.25929999351501465},{"id":"https://openalex.org/C148934300","wikidata":"https://www.wikidata.org/wiki/Q40998","display_name":"Phonology","level":2,"score":0.2565000057220459},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.25540000200271606}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.17105","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.17105","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.17105","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.17105","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Tokenization":[0],"is":[1],"the":[2,14,45,64,73,80],"first":[3],"step":[4],"in":[5],"every":[6],"language":[7],"model":[8],"(LM),":[9],"yet":[10],"it":[11],"never":[12],"takes":[13],"sounds":[15],"of":[16,35,47,84],"words":[17],"into":[18,118],"account.":[19],"We":[20],"investigate":[21],"how":[22],"tokenization":[23,42,78],"influences":[24],"text-only":[25],"LMs'":[26],"ability":[27],"to":[28,121],"represent":[29],"phonological":[30,56,94,116],"knowledge.":[31],"Through":[32],"a":[33,69,76,97,109],"series":[34],"probing":[36],"experiments,":[37],"we":[38,62,107],"show":[39],"that":[40,71,88,114],"subword-based":[41],"systematically":[43],"weakens":[44],"encoding":[46],"both":[48],"local":[49],"(e.g.,":[50,54],"rhyme)":[51],"and":[52,79,86,132,138,143],"global":[53],"syllabification)":[55],"features.":[57],"To":[58,103],"quantify":[59],"this":[60],"effect,":[61],"introduce":[63],"syllabification-tokenization":[65],"alignment":[66],"distance":[67],"(STAD),":[68],"metric":[70],"measures":[72],"misalignment":[74,90],"between":[75],"model's":[77],"natural":[81],"syllable":[82],"boundaries":[83],"words,":[85],"find":[87],"higher":[89],"correlates":[91],"with":[92,136],"poorer":[93],"representations,":[95],"providing":[96],"simple":[98],"diagnostic":[99],"for":[100],"phonology-aware":[101],"tokenization.":[102],"address":[104],"these":[105],"limitations,":[106],"propose":[108],"lightweight":[110],"IPA-based":[111],"fine-tuning":[112],"method":[113],"infuses":[115],"awareness":[117],"LMs,":[119],"leading":[120],"consistent":[122],"improvements":[123],"across":[124],"three":[125],"phonology-related":[126],"tasks":[127],"while":[128],"largely":[129],"preserving":[130],"math":[131],"general":[133],"reasoning":[134],"ability,":[135],"1.1\\%":[137],"0.9\\%":[139],"drops":[140],"on":[141],"GSM8K":[142],"MMLU,":[144],"respectively.":[145]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-22T00:00:00"}