{"id":"https://openalex.org/W7021066849","doi":null,"title":"N-Gram Learning and Pretraining Dynamics in Transformer Language Models","display_name":"N-Gram Learning and Pretraining Dynamics in Transformer Language Models","alternate_titles":[],"publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W7021066849"},"language":"en","primary_location":{"id":"pmh:oai:escholarship.org:ark:/13030/qt7gh8b2z9","doi":null,"is_oa":true,"landing_page_url":"https://escholarship.org/uc/item/7gh8b2z9","pdf_url":"https://escholarship.org/content/qt7gh8b2z9/qt7gh8b2z9.pdf","source":{"id":"https://openalex.org/S4306400115","display_name":"eScholarship (California Digital Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801248553","host_organization_name":"California Digital Library","host_organization_lineage":["https://openalex.org/I2801248553"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"etd"},"parent_works":[],"lineage":["https://openalex.org/W7021066849"],"child_works_count":0,"type":"dissertation","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://escholarship.org/content/qt7gh8b2z9/qt7gh8b2z9.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Chang, Tyler","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I188497080","display_name":"Carleton College","ror":"https://ror.org/03jep7677","country_code":"US","type":"education","lineage":["https://openalex.org/I188497080"]}],"countries":["US"],"is_corresponding":true,"contributor_roles":[],"raw_author_name":"Chang, Tyler","raw_affiliation_strings":["BA in Cognitive Science , Carleton College 2025"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"BA in Cognitive Science , Carleton College 2025","institution_ids":["https://openalex.org/I188497080"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I188497080"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8535000085830688,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8535000085830688,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.11710000038146973,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10730","display_name":"Language Development and Disorders","score":0.004699999932199717,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer-based-language-models","display_name":"transformer language models","score":0.734000027179718},{"id":"https://openalex.org/keywords/next-token-prediction","display_name":"next-token prediction","score":0.8090000152587891},{"id":"https://openalex.org/keywords/word-frequency","display_name":"word frequency","score":0.5400000214576721},{"id":"https://openalex.org/keywords/child-language-acquisition","display_name":"child language acquisition","score":0.7110000252723694},{"id":"https://openalex.org/keywords/activation-space","display_name":"activation space","score":0.7350000143051147},{"id":"https://openalex.org/keywords/distributional-statistics","display_name":"distributional statistics","score":0.6499999761581421}],"concepts":[{"id":"https://openalex.org/C108757681","wikidata":"https://www.wikidata.org/wiki/Q2773912","display_name":"Bigram","level":3,"score":0.9757000207901001},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.7728000283241272},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7653999924659729},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6003000140190125},{"id":"https://openalex.org/C39608478","wikidata":"https://www.wikidata.org/wiki/Q5015979","display_name":"Cache language model","level":5,"score":0.5817999839782715},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5551000237464905},{"id":"https://openalex.org/C74672266","wikidata":"https://www.wikidata.org/wiki/Q815859","display_name":"Language acquisition","level":2,"score":0.5400000214576721},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.53329998254776},{"id":"https://openalex.org/C129792486","wikidata":"https://www.wikidata.org/wiki/Q1050419","display_name":"Language identification","level":3,"score":0.51419997215271},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4803999960422516},{"id":"https://openalex.org/C94922259","wikidata":"https://www.wikidata.org/wiki/Q33215","display_name":"Constructed language","level":2,"score":0.39989998936653137},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34610000252723694},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.33880001306533813},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.3352999985218048},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.33410000801086426},{"id":"https://openalex.org/C83479923","wikidata":"https://www.wikidata.org/wiki/Q2063748","display_name":"Universal Networking Language","level":4,"score":0.3271999955177307},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.3208000063896179},{"id":"https://openalex.org/C2993724205","wikidata":"https://www.wikidata.org/wiki/Q315","display_name":"Human language","level":2,"score":0.3199999928474426},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.3160000145435333},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3005000054836273},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.29829999804496765},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.2937999963760376},{"id":"https://openalex.org/C129353971","wikidata":"https://www.wikidata.org/wiki/Q5156949","display_name":"Comprehension approach","level":3,"score":0.2831999957561493},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.2827000021934509},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.2816999852657318},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2741999924182892},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.2741999924182892},{"id":"https://openalex.org/C32254414","wikidata":"https://www.wikidata.org/wiki/Q4724364","display_name":"Algorithmic learning theory","level":3,"score":0.25600001215934753},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.25290000438690186}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:escholarship.org:ark:/13030/qt7gh8b2z9","doi":null,"is_oa":true,"landing_page_url":"https://escholarship.org/uc/item/7gh8b2z9","pdf_url":"https://escholarship.org/content/qt7gh8b2z9/qt7gh8b2z9.pdf","source":{"id":"https://openalex.org/S4306400115","display_name":"eScholarship (California Digital Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801248553","host_organization_name":"California Digital Library","host_organization_lineage":["https://openalex.org/I2801248553"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"etd"}],"best_oa_location":{"id":"pmh:oai:escholarship.org:ark:/13030/qt7gh8b2z9","doi":null,"is_oa":true,"landing_page_url":"https://escholarship.org/uc/item/7gh8b2z9","pdf_url":"https://escholarship.org/content/qt7gh8b2z9/qt7gh8b2z9.pdf","source":{"id":"https://openalex.org/S4306400115","display_name":"eScholarship (California Digital Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801248553","host_organization_name":"California Digital Library","host_organization_lineage":["https://openalex.org/I2801248553"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"etd"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7021066849.pdf","grobid_xml":"https://content.openalex.org/works/W7021066849.grobid-xml"},"referenced_works_count":115,"referenced_works":["https://openalex.org/W1566289585","https://openalex.org/W1880262756","https://openalex.org/W1951216520","https://openalex.org/W1974991592","https://openalex.org/W1976234022","https://openalex.org/W1993984071","https://openalex.org/W1998151045","https://openalex.org/W2095689128","https://openalex.org/W2098701576","https://openalex.org/W2100407944","https://openalex.org/W2105834992","https://openalex.org/W2111945016","https://openalex.org/W2119728020","https://openalex.org/W2134237567","https://openalex.org/W2138410680","https://openalex.org/W2142102918","https://openalex.org/W2154600605","https://openalex.org/W2164418233","https://openalex.org/W2180912620","https://openalex.org/W2396361046","https://openalex.org/W2605959375","https://openalex.org/W2763088512","https://openalex.org/W2787345674","https://openalex.org/W2787560479","https://openalex.org/W2795342569","https://openalex.org/W2795355797","https://openalex.org/W2884554299","https://openalex.org/W2885185669","https://openalex.org/W2888922637","https://openalex.org/W2921890305","https://openalex.org/W2944797580","https://openalex.org/W2950050806","https://openalex.org/W2950976310","https://openalex.org/W2953380194","https://openalex.org/W2963341956","https://openalex.org/W2965647350","https://openalex.org/W2979826702","https://openalex.org/W2995107071","https://openalex.org/W3006881356","https://openalex.org/W3018827121","https://openalex.org/W3022647948","https://openalex.org/W3030163527","https://openalex.org/W3034253961","https://openalex.org/W3034510440","https://openalex.org/W3034723486","https://openalex.org/W3088647504","https://openalex.org/W3092592453","https://openalex.org/W3117576675","https://openalex.org/W3119686325","https://openalex.org/W3120253119","https://openalex.org/W3122890974","https://openalex.org/W3135131435","https://openalex.org/W3168154341","https://openalex.org/W3172136722","https://openalex.org/W3172673629","https://openalex.org/W3174617925","https://openalex.org/W3175560839","https://openalex.org/W3183430109","https://openalex.org/W3198002980","https://openalex.org/W3198757395","https://openalex.org/W3199373335","https://openalex.org/W3200809495","https://openalex.org/W3202180397","https://openalex.org/W3205187613","https://openalex.org/W3205328383","https://openalex.org/W3213280751","https://openalex.org/W3213448373","https://openalex.org/W4226399820","https://openalex.org/W4287890137","https://openalex.org/W4297412003","https://openalex.org/W4309419356","https://openalex.org/W4312205996","https://openalex.org/W4327526612","https://openalex.org/W4327526719","https://openalex.org/W4327810158","https://openalex.org/W4361766487","https://openalex.org/W4362655426","https://openalex.org/W4362696539","https://openalex.org/W4377121468","https://openalex.org/W4385570738","https://openalex.org/W4385571329","https://openalex.org/W4385571791","https://openalex.org/W4385572928","https://openalex.org/W4385572950","https://openalex.org/W4385573636","https://openalex.org/W4386437578","https://openalex.org/W4386794314","https://openalex.org/W4388691863","https://openalex.org/W4389519056","https://openalex.org/W4389520380","https://openalex.org/W4390041933","https://openalex.org/W4391212816","https://openalex.org/W4391462055","https://openalex.org/W4391670818","https://openalex.org/W4392240262","https://openalex.org/W4392935324","https://openalex.org/W4393247942","https://openalex.org/W4401042778","https://openalex.org/W4402670697","https://openalex.org/W4404132457","https://openalex.org/W4404783038","https://openalex.org/W4404783649","https://openalex.org/W4411119427","https://openalex.org/W4415796200","https://openalex.org/W4415797427","https://openalex.org/W4415797858","https://openalex.org/W4417091950","https://openalex.org/W7018686672","https://openalex.org/W7133211372","https://openalex.org/W7133218086","https://openalex.org/W7133224126","https://openalex.org/W7133224150","https://openalex.org/W7133225605","https://openalex.org/W7133236469","https://openalex.org/W7133262701"],"related_works":["https://openalex.org/W4404132457","https://openalex.org/W4404783038","https://openalex.org/W3202180397","https://openalex.org/W6979909690","https://openalex.org/W4416872443","https://openalex.org/W4283809686","https://openalex.org/W4416368807","https://openalex.org/W4395443613","https://openalex.org/W7166565611","https://openalex.org/W4389518431"],"abstract_inverted_index":{"Transformer":[0],"language":[1,16,29,85,94,119,128,182,198,215],"models":[2,30,95,163,168],"have":[3,169],"received":[4],"unprecedented":[5],"attention":[6],"in":[7,39,113,118,123,161,186,227],"recent":[8],"years":[9],"due":[10],"to":[11,33,48,55,68,181],"impressive":[12],"performance":[13,184],"on":[14,79,147,171],"natural":[15],"tasks,":[17],"but":[18,121],"how":[19,50],"they":[20,191],"acquire":[21],"these":[22,205,228],"capabilities":[23],"is":[24],"largely":[25],"unknown.":[26],"During":[27],"pretraining,":[28],"are":[31,67,179],"trained":[32,188],"predict":[34],"next":[35],"tokens":[36],"(e.g.":[37],"words)":[38],"text,":[40],"and":[41,59,84,150,190,214],"the":[42,80,108,162,166],"field":[43],"of":[44,64,111,116,141,196],"pretraining":[45],"dynamics":[46],"aims":[47],"demonstrate":[49,69,207],"this":[51,65],"learning":[52,57,87,130,134],"objective":[53],"leads":[54],"consistent":[56],"patterns":[58],"mechanisms.":[60],"The":[61],"primary":[62],"contributions":[63],"dissertation":[66],"relationships":[70],"between":[71,211],"n-gram":[72,133,144,212],"probabilities":[73,213],"(next":[74],"token":[75],"predictions":[76,145,159],"based":[77,146],"only":[78],"previous":[81],"n-1":[82],"tokens)":[83],"model":[86,129,216],"during":[88],"pretraining.":[89],"I":[90,125],"first":[91],"show":[92],"that":[93,156,219],"learn":[96],"words":[97],"differently":[98],"from":[99,172],"human":[100],"children;":[101],"word":[102],"frequencies":[103],"(unigram":[104],"probabilities)":[105],"account":[106],"for":[107,135],"vast":[109],"majority":[110],"variance":[112],"words\u2019":[114],"ages":[115],"acquisition":[117],"models,":[120,189],"not":[122],"children.":[124],"then":[126,138],"characterize":[127],"as":[131],"early":[132],"increasing":[136],"n,":[137],"gradual":[139],"refinement":[140],"low":[142],"probability":[143],"longer":[148],"contexts":[149],"more":[151,229],"nuanced":[152],"capabilities.":[153],"Mechanistically,":[154],"subnetworks":[155,178],"make":[157],"bigram":[158,174,177],"exist":[160],"long":[164],"after":[165],"full":[167],"moved":[170],"simple":[173],"predictions.":[175],"These":[176],"critical":[180],"modeling":[183],"even":[185],"fully":[187],"recreate":[192],"several":[193],"key":[194],"properties":[195],"a":[197,208],"model\u2019s":[199],"activation":[200],"space":[201],"across":[202],"layers.":[203],"Together,":[204],"results":[206],"close":[209],"relationship":[210],"learning,":[217],"suggesting":[218],"traditional":[220],"distributional":[221],"statistics":[222],"still":[223],"play":[224],"some":[225],"role":[226],"complex":[230],"models.":[231]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7021066849","counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2025-10-10T00:00:00"}
