{"id":"https://openalex.org/W7152970037","doi":"https://doi.org/10.48550/arxiv.2604.08510","title":"What do Language Models Learn and When? The Implicit Curriculum Hypothesis","display_name":"What do Language Models Learn and When? The Implicit Curriculum Hypothesis","publication_year":2026,"publication_date":"2026-04-09","ids":{"openalex":"https://openalex.org/W7152970037","doi":"https://doi.org/10.48550/arxiv.2604.08510"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.08510","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.08510","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.08510","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055560341","display_name":"Emmy Liu","orcid":"https://orcid.org/0000-0003-1287-5712"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Emmy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133346819","display_name":"Kaiser Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Kaiser","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133382836","display_name":"Millicent Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Millicent","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133387690","display_name":"Isabelle Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Isabelle","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092050382","display_name":"Lindia Tjuatja","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tjuatja, Lindia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133360062","display_name":"Jen-tse Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Jen-tse","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133384273","display_name":"Graham Neubig","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Neubig, Graham","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.15080000460147858,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.15080000460147858,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.11069999635219574,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.10819999873638153,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5831000208854675},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5442000031471252},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.5339000225067139},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.5067999958992004},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4521999955177307},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.43479999899864197},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.4334000051021576},{"id":"https://openalex.org/keywords/task-analysis","display_name":"Task analysis","score":0.3614000082015991}],"concepts":[{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5831000208854675},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.572700023651123},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5442000031471252},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.5339000225067139},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.5094000101089478},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.5067999958992004},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5016999840736389},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4521999955177307},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4498000144958496},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.43479999899864197},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4334000051021576},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3614000082015991},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3215000033378601},{"id":"https://openalex.org/C2777379011","wikidata":"https://www.wikidata.org/wiki/Q938545","display_name":"Implicit learning","level":3,"score":0.3181999921798706},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.30160000920295715},{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.296099990606308},{"id":"https://openalex.org/C121375916","wikidata":"https://www.wikidata.org/wiki/Q936559","display_name":"Principle of compositionality","level":2,"score":0.2937000095844269},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.2840999960899353},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.2736999988555908},{"id":"https://openalex.org/C16910744","wikidata":"https://www.wikidata.org/wiki/Q7705759","display_name":"Test data","level":2,"score":0.2619999945163727},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.260699987411499},{"id":"https://openalex.org/C203702819","wikidata":"https://www.wikidata.org/wiki/Q17146953","display_name":"Logical data model","level":3,"score":0.25600001215934753},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.2535000145435333}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.08510","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.08510","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.08510","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.08510","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.8862724900245667,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,64,111,221],"(LLMs)":[3],"can":[4,173],"perform":[5],"remarkably":[6],"complex":[7],"tasks,":[8,90],"yet":[9],"the":[10,52,163,176,185],"fine-grained":[11],"details":[12],"of":[13,75,109,165,179,187],"how":[14,30],"these":[15,89,199],"capabilities":[16],"emerge":[17,131,212],"during":[18],"pretraining":[19,56,188,203],"remain":[20],"poorly":[21],"understood.":[22],"Scaling":[23],"laws":[24],"on":[25],"validation":[26],"loss":[27,208],"tell":[28],"us":[29],"much":[31],"a":[32,58,73,214],"model":[33,97,123,145],"improves":[34],"with":[35,148],"additional":[36],"compute,":[37],"but":[38],"not":[39],"what":[40],"skills":[41,211],"it":[42],"acquires":[43],"in":[44,144,159,213],"which":[45],"order.":[46],"To":[47],"remedy":[48],"this,":[49],"we":[50,91,137,172],"propose":[51],"Implicit":[53],"Curriculum":[54],"Hypothesis:":[55],"follows":[57],"compositional":[59,182,215],"and":[60,65,86,125,222],"predictable":[61],"curriculum":[62],"across":[63,95,121,192,220],"data":[66],"mixtures.":[67],"We":[68,104],"test":[69],"this":[70,140],"by":[71],"designing":[72],"suite":[74],"simple,":[76],"composable":[77],"tasks":[78,128,147,183],"spanning":[79,99],"retrieval,":[80],"morphological":[81],"transformations,":[82],"coreference,":[83],"logical":[84],"reasoning,":[85],"mathematics.":[87],"Using":[88],"track":[92],"emergence":[93,107],"points":[94],"four":[96],"families":[98],"sizes":[100],"from":[101,168,224],"410M-13B":[102],"parameters.":[103],"find":[105,138],"that":[106,126,139,202,217],"orderings":[108],"when":[110],"reach":[112],"fixed":[113],"accuracy":[114],"thresholds":[115],"are":[116],"strikingly":[117],"consistent":[118,219],"($\u03c1=":[119],".81$":[120],"45":[122],"pairs),":[124],"composite":[127],"most":[129],"often":[130],"after":[132],"their":[133,225],"component":[134],"tasks.":[135],"Furthermore,":[136],"structure":[141],"is":[142,204,218],"encoded":[143],"representations:":[146],"similar":[149,157],"function":[150],"vector":[151],"representations":[152,166],"also":[153],"tend":[154],"to":[155],"follow":[156],"trajectories":[158,178],"training.":[160],"By":[161],"using":[162],"space":[164],"derived":[167],"our":[169],"task":[170],"set,":[171],"effectively":[174],"predict":[175],"training":[177],"simple":[180],"held-out":[181],"throughout":[184],"course":[186],"($R^2":[189],"=":[190],".68$-$.84$":[191],"models)":[193],"without":[194],"previously":[195],"evaluating":[196],"them.":[197],"Together,":[198],"results":[200],"suggest":[201],"more":[205],"structured":[206],"than":[207],"curves":[209],"reveal:":[210],"order":[216],"readable":[223],"internals.":[226]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-11T00:00:00"}