{"id":"https://openalex.org/W7164046008","doi":"https://doi.org/10.48550/arxiv.2606.08562","title":"Inside the LLM Word Factory","display_name":"Inside the LLM Word Factory","publication_year":2026,"publication_date":"2026-06-07","ids":{"openalex":"https://openalex.org/W7164046008","doi":"https://doi.org/10.48550/arxiv.2606.08562"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.08562","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08562","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.08562","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138235712","display_name":"Benzi Busigin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Busigin, Benzi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138206297","display_name":"Yuval Pinter","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pinter, Yuval","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.25119999051094055,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.25119999051094055,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1777999997138977,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10465","display_name":"Neurobiology of Language and Bilingualism","score":0.1671999990940094,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6159999966621399},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5544000267982483},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.5306000113487244},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.5023000240325928},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.4426000118255615},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4156999886035919},{"id":"https://openalex.org/keywords/process-modeling","display_name":"Process modeling","score":0.37279999256134033}],"concepts":[{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6159999966621399},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6011999845504761},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.567300021648407},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5544000267982483},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5512999892234802},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5306000113487244},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.5023000240325928},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.4426000118255615},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4156999886035919},{"id":"https://openalex.org/C76956256","wikidata":"https://www.wikidata.org/wiki/Q27610560","display_name":"Process modeling","level":3,"score":0.37279999256134033},{"id":"https://openalex.org/C40149104","wikidata":"https://www.wikidata.org/wiki/Q5620977","display_name":"Factory (object-oriented programming)","level":2,"score":0.328000009059906},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.319599986076355},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.29170000553131104},{"id":"https://openalex.org/C174998907","wikidata":"https://www.wikidata.org/wiki/Q357662","display_name":"Work in process","level":2,"score":0.2874999940395355},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2750000059604645},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2567000091075897},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.2563000023365021},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.25209999084472656}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.08562","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08562","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.08562","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08562","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Transformer":[0],"language":[1,11],"models":[2,23,122,142,151],"process":[3,21,56,90,168],"input":[4],"provided":[5],"as":[6],"subword":[7],"fragments,":[8],"but":[9,48,126],"natural":[10],"semantics":[12],"usually":[13],"rely":[14],"on":[15,135,170,179],"word-level":[16,31],"concepts.":[17],"Detokenization":[18],"is":[19],"the":[20,51,55,76,108,113,127,136,163,166,180],"where":[22],"reconcile":[24],"these":[25],"two":[26],"facts,":[27],"aggregating":[28],"subwords":[29],"into":[30,65],"representations":[32],"through":[33],"their":[34],"computation.":[35],"Prior":[36],"work":[37],"has":[38],"found":[39],"that":[40,74],"this":[41],"takes":[42,132],"place":[43,133],"mostly":[44],"in":[45,70,85],"early-to-middle":[46],"layers,":[47,148],"so":[49],"far":[50],"exact":[52],"mechanics":[53],"of":[54,78,138,165,182],"have":[57],"not":[58],"been":[59],"pinned":[60],"down.":[61],"We":[62],"venture":[63],"deep":[64],"detokenization":[66,84,167],"using":[67,102],"activation":[68],"patching":[69],"controlled":[71],"paired":[72],"experiments":[73],"isolate":[75],"contribution":[77],"different":[79],"model":[80],"components,":[81],"localizing":[82],"English":[83],"Llama2-7B":[86],"to":[87,120,146,154],"a":[88,96,159],"two-stage":[89,117],"at":[91,175],"Layer":[92],"1.":[93],"Attention":[94],"transmits":[95],"token-specific":[97],"signal":[98],"from":[99,123],"nonfinal":[100],"subwords,":[101],"sequential":[103],"relays":[104],"if":[105],"necessary,":[106],"while":[107,149],"MLP":[109],"composes":[110],"it":[111,131],"with":[112],"local":[114],"embedding.":[115],"This":[116],"structure":[118],"generalizes":[119],"twelve":[121],"eight":[124],"families,":[125],"depth":[128],"over":[129,144],"which":[130],"depends":[134],"flavor":[137],"positional":[139],"encoding:":[140],"RoPE-based":[141],"detokenize":[143],"1":[145],"5":[147,153],"learned-absolute":[150],"take":[152],"10.":[155],"Finally,":[156],"we":[157],"provide":[158],"probe":[160],"for":[161],"determining":[162],"success":[164],"based":[169],"early-layer":[171],"activations":[172],"alone,":[173],"performing":[174],"0.94-0.97":[176],"AUROC":[177],"depending":[178],"amount":[181],"context.":[183]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-10T00:00:00"}