{"id":"https://openalex.org/W7155030754","doi":"https://doi.org/10.48550/arxiv.2604.16656","title":"Defragmenting Language Models: An Interpretability-based Approach for Vocabulary Expansion","display_name":"Defragmenting Language Models: An Interpretability-based Approach for Vocabulary Expansion","publication_year":2026,"publication_date":"2026-04-17","ids":{"openalex":"https://openalex.org/W7155030754","doi":"https://doi.org/10.48550/arxiv.2604.16656"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.16656","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.16656","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.16656","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078270542","display_name":"Maitrey Mehta","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mehta, Maitrey","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109481132","display_name":"Nishant Subramani","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Subramani, Nishant","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134210494","display_name":"Zhichao Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Zhichao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134138056","display_name":"Ashim Gupta","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gupta, Ashim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5013135203","display_name":"Vivek Srikumar","orcid":"https://orcid.org/0000-0003-0419-6568"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Srikumar, Vivek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.7901999950408936,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.7901999950408936,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.07020000368356705,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.06199999898672104,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.6676999926567078},{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.4666999876499176},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.35740000009536743},{"id":"https://openalex.org/keywords/merge","display_name":"Merge (version control)","score":0.35440000891685486},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.3538999855518341},{"id":"https://openalex.org/keywords/negation","display_name":"Negation","score":0.3312000036239624},{"id":"https://openalex.org/keywords/initialization","display_name":"Initialization","score":0.3239000141620636}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7785999774932861},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.6676999926567078},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.567300021648407},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5078999996185303},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.4666999876499176},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.35740000009536743},{"id":"https://openalex.org/C197129107","wikidata":"https://www.wikidata.org/wiki/Q1921621","display_name":"Merge (version control)","level":2,"score":0.35440000891685486},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3538999855518341},{"id":"https://openalex.org/C2185349","wikidata":"https://www.wikidata.org/wiki/Q190558","display_name":"Negation","level":2,"score":0.3312000036239624},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.3239000141620636},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.32199999690055847},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2987000048160553},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.2854999899864197},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.28200000524520874},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.2773999869823456},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.26510000228881836},{"id":"https://openalex.org/C540372491","wikidata":"https://www.wikidata.org/wiki/Q82622","display_name":"Punctuation","level":2,"score":0.26420000195503235},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.2619999945163727}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.16656","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.16656","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.16656","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.16656","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"All":[0],"languages":[1,33,56,189],"are":[2,10,16],"equal;":[3],"when":[4],"it":[5],"comes":[6],"to":[7,28,49,144,149,223],"tokenization,":[8],"some":[9],"more":[11],"equal":[12],"than":[13],"others.":[14],"Tokens":[15],"the":[17,22,51,90,115,138,171,196,226,234],"hidden":[18],"currency":[19],"that":[20,65,83,159],"dictate":[21],"cost":[23],"and":[24,99,124,132,157,243],"latency":[25],"of":[26,47,141,198,217,229,236,248],"access":[27],"contemporary":[29],"LLMs.":[30,76],"However,":[31],"many":[32,55],"written":[34,190],"in":[35,54,73,114,191,214],"non-Latin":[36,192],"scripts":[37],"observe":[38],"a":[39,104,163],"poor":[40],"exchange":[41],"rate:":[42],"LLMs":[43],"take":[44],"several":[45,188],"multiples":[46],"tokens":[48,207],"encode":[50],"same":[52],"information":[53],"as":[57,69,155],"they":[58],"do":[59],"for":[60,173,187],"English.":[61],"Our":[62],"analysis":[63,216,247],"reveals":[64],"this":[66,94,218],"issue,":[67],"known":[68],"'token":[70],"over-fragmentation',":[71],"persists":[72],"modern":[74],"open-weight":[75],"The":[77],"standard":[78],"remedy":[79],"is":[80],"vocabulary":[81,102,116,147],"expansion":[82,117],"adds":[84],"target":[85],"language":[86],"items":[87,120,148],"missing":[88],"from":[89],"model's":[91],"vocabulary.":[92],"In":[93],"work,":[95],"we":[96,122,127,136,169,220,244],"comprehensively":[97],"study":[98],"advance":[100],"interpretability-based":[101,160,174,230],"expansion,":[103],"new":[105],"research":[106],"direction.":[107],"We":[108,194,232],"focus":[109],"on":[110],"two":[111],"core":[112],"decisions":[113],"process:":[118],"What":[119],"should":[121,126],"add?":[123],"How":[125],"initialize":[128],"their":[129],"corresponding":[130],"input":[131],"output":[133],"embeddings?":[134],"First,":[135],"question":[137],"conventional":[139],"use":[140],"frequency-based":[142],"methods":[143,161,186],"choose":[145],"candidate":[146],"add":[150],"(a":[151],"decision":[152],"long":[153],"treated":[154],"settled),":[156],"show":[158],"offer":[162],"superior":[164],"performance-token":[165],"efficiency":[166,227],"trade-off.":[167],"Next,":[168],"strengthen":[170],"case":[172],"embedding":[175],"initialization":[176,185],"by":[177],"showing":[178],"large":[179],"gains":[180],"(~20":[181],"pts)":[182],"over":[183],"baseline":[184],"scripts.":[193],"identify":[195],"phenomenon":[197],"\"subword":[199],"detokenization\"":[200],"where":[201],"models":[202],"progressively":[203],"merge":[204],"fragmented":[205],"subword":[206],"into":[208],"larger":[209],"subwords":[210],"across":[211],"layers.":[212],"Grounded":[213],"our":[215],"phenomenon,":[219],"propose":[221],"FragMend":[222,237],"further":[224],"push":[225],"ceiling":[228],"expansion.":[231],"validate":[233],"effectiveness":[235],"through":[238],"comparison":[239],"against":[240],"strong":[241],"baselines":[242],"present":[245],"extensive":[246],"its":[249],"design":[250],"choices.":[251]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-22T00:00:00"}