{"id":"https://openalex.org/W4403564552","doi":"https://doi.org/10.48550/arxiv.2410.09303","title":"Exact Byte-Level Probabilities from Tokenized Language Models for FIM-Tasks and Model Ensembles","display_name":"Exact Byte-Level Probabilities from Tokenized Language Models for FIM-Tasks and Model Ensembles","publication_year":2024,"publication_date":"2024-10-11","ids":{"openalex":"https://openalex.org/W4403564552","doi":"https://doi.org/10.48550/arxiv.2410.09303"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.09303","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.09303","pdf_url":"https://arxiv.org/pdf/2410.09303","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.09303","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085652556","display_name":"Buu Phan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Phan, Buu","raw_affiliation_strings":["University of Toronto ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto ,","institution_ids":["https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030666794","display_name":"Brandon Amos","orcid":"https://orcid.org/0000-0003-2884-0119","observed_orcids":["https://orcid.org/0000-0003-2884-0119"]},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amos, Brandon","raw_affiliation_strings":["Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta AI","institution_ids":["https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Gat, Itai","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gat, Itai","raw_affiliation_strings":["Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta AI","institution_ids":["https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099497279","display_name":"Marton Havasi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Havasi, Marton","raw_affiliation_strings":["Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta AI","institution_ids":["https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029739727","display_name":"Matthew J. Muckley","orcid":"https://orcid.org/0000-0002-6525-8817","observed_orcids":["https://orcid.org/0000-0002-6525-8817"]},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Muckley, Matthew","raw_affiliation_strings":["Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta AI","institution_ids":["https://openalex.org/I4210114444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5058031547","display_name":"Karen Ullrich","orcid":"https://orcid.org/0000-0002-9811-684X","observed_orcids":["https://orcid.org/0000-0002-9811-684X"]},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ullrich, Karen","raw_affiliation_strings":["Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta AI","institution_ids":["https://openalex.org/I4210114444"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7440999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7440999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2425999939441681,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.0017999999690800905,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ensemble-learning","display_name":"ensemble learning","score":0.859000027179718}],"concepts":[{"id":"https://openalex.org/C43364308","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Byte","level":2,"score":0.6724292039871216},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5678308010101318},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35988685488700867},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.35537150502204895},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3425344228744507},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.28860121965408325},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1917164921760559}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.09303","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.09303","pdf_url":"https://arxiv.org/pdf/2410.09303","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.09303","doi":"https://doi.org/10.48550/arxiv.2410.09303","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.09303","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.09303","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.09303","pdf_url":"https://arxiv.org/pdf/2410.09303","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403564552.pdf","grobid_xml":"https://content.openalex.org/works/W4403564552.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4400024841","https://openalex.org/W4415311598","https://openalex.org/W7220729864","https://openalex.org/W4415795367","https://openalex.org/W4414682129","https://openalex.org/W4405468714","https://openalex.org/W7127320815","https://openalex.org/W7116131315","https://openalex.org/W4394838906","https://openalex.org/W4415143708"],"abstract_inverted_index":{"Tokenization":[0],"is":[1,220],"associated":[2],"with":[3,40,141],"many":[4],"poorly":[5],"understood":[6],"shortcomings":[7],"in":[8,173,201,214],"language":[9],"models":[10,39,53,209],"(LMs),":[11],"yet":[12],"remains":[13],"an":[14],"important":[15],"component":[16],"for":[17],"long":[18],"sequence":[19],"scaling":[20],"purposes.":[21],"This":[22],"work":[23],"studies":[24],"how":[25],"tokenization":[26,113],"impacts":[27],"model":[28,149,186,190],"performance":[29,167,203],"by":[30],"analyzing":[31],"and":[32,97,148,169,217],"comparing":[33],"the":[34,51,61,82,93,180],"stochastic":[35],"behavior":[36],"of":[37,128],"tokenized":[38,129],"their":[41,57],"byte-level,":[42],"or":[43,119],"token-free,":[44],"counterparts.":[45],"We":[46,136],"discover":[47],"that,":[48],"even":[49],"when":[50],"two":[52,142],"are":[54],"statistically":[55,132],"equivalent,":[56],"predictive":[58],"distributions":[59],"over":[60,207],"next":[62],"byte":[63],"can":[64],"be":[65],"substantially":[66],"different,":[67],"a":[68,86,90,107,192],"phenomenon":[69],"we":[70,80,105],"term":[71],"as":[72],"``tokenization":[73],"bias''.":[74],"To":[75],"fully":[76],"characterize":[77],"this":[78,103,124],"phenomenon,":[79],"introduce":[81],"Byte-Token":[83],"Representation":[84],"Lemma,":[85],"framework":[87],"that":[88,111],"establishes":[89],"mapping":[91],"between":[92],"learned":[94],"token":[95,182],"distribution":[96],"its":[98,138],"equivalent":[99,133],"byte-level":[100],"distribution.":[101],"From":[102],"result,":[104],"develop":[106],"next-byte":[108],"sampling":[109],"algorithm":[110],"eliminates":[112],"bias":[114],"without":[115],"requiring":[116],"further":[117],"training":[118],"optimization.":[120],"In":[121,151],"other":[122],"words,":[123],"enables":[125,197],"zero-shot":[126],"conversion":[127],"LMs":[130],"into":[131],"token-free":[134],"ones.":[135],"demonstrate":[137],"broad":[139],"applicability":[140],"use":[143],"cases:":[144],"fill-in-the-middle":[145],"(FIM)":[146],"tasks":[147,153],"ensembles.":[150],"FIM":[152,174],"where":[154,188],"input":[155],"prompts":[156],"may":[157],"terminate":[158],"mid-token,":[159],"leading":[160],"to":[161,205],"out-of-distribution":[162],"tokenization,":[163],"our":[164,195],"method":[165],"mitigates":[166],"degradation":[168],"achieves":[170],"18%":[171],"improvement":[172],"coding":[175],"benchmarks,":[176],"while":[177],"consistently":[178],"outperforming":[179],"standard":[181,212],"healing":[183],"fix.":[184],"For":[185],"ensembles":[187],"each":[189],"employs":[191],"distinct":[193],"vocabulary,":[194],"approach":[196],"seamless":[198],"integration,":[199],"resulting":[200],"improved":[202],"up":[204],"3.7%":[206],"individual":[208],"across":[210],"various":[211],"baselines":[213],"reasoning,":[215],"knowledge,":[216],"coding.":[218],"Code":[219],"available":[221],"at:":[222],"https://github.com/facebookresearch/Exact-Byte-Level-Probabilities-from-Tokenized-LMs":[223]},"counts_by_year":[],"updated_date":"2026-10-02T10:15:12.560889","created_date":"2024-10-20T00:00:00"}