{"id":"https://openalex.org/W7204383257","doi":"https://doi.org/10.48550/arxiv.2605.22769","title":"Understanding Data Temporality Impact on Large Language Models Pre-training","display_name":"Understanding Data Temporality Impact on Large Language Models Pre-training","publication_year":2026,"publication_date":"2026-05-21","ids":{"openalex":"https://openalex.org/W7204383257","doi":"https://doi.org/10.48550/arxiv.2605.22769"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2605.22769","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2605.22769","pdf_url":"https://arxiv.org/pdf/2605.22769","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2605.22769","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120663902","display_name":"Hippolyte Pilchen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pilchen, Hippolyte","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041309234","display_name":"Romain Fabre","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fabre, Romain","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114452726","display_name":"Franck Signe Talla","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Talla, Franck Signe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113908330","display_name":"Patrick P\u00e9rez","orcid":"https://orcid.org/0000-0002-8124-1206","observed_orcids":["https://orcid.org/0000-0002-8124-1206"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Perez, Patrick","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5069316249","display_name":"\u00c9douard Grave","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Grave, Edouard","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0044999998062849045,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.003700000001117587,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9470000267028809},{"id":"https://openalex.org/keywords/temporal-grounding","display_name":"temporal grounding","score":0.828000009059906},{"id":"https://openalex.org/keywords/continual-learning","display_name":"continual learning","score":0.8360000252723694},{"id":"https://openalex.org/keywords/factual-grounding","display_name":"factual grounding","score":0.3190000057220459}],"concepts":[{"id":"https://openalex.org/C141679102","wikidata":"https://www.wikidata.org/wiki/Q3517747","display_name":"Temporality","level":2,"score":0.7261000275611877},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.5127999782562256},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5065000057220459},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.36660000681877136},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.32359999418258667},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.3107999861240387},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.30799999833106995},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.3052000105381012},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2969000041484833},{"id":"https://openalex.org/C2780878386","wikidata":"https://www.wikidata.org/wiki/Q1659648","display_name":"Visual language","level":2,"score":0.2879999876022339},{"id":"https://openalex.org/C2778137410","wikidata":"https://www.wikidata.org/wiki/Q2732820","display_name":"Government (linguistics)","level":2,"score":0.28679999709129333},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.27300000190734863},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2700999975204468},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.26739999651908875},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.26159998774528503},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2556000053882599},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.2506999969482422},{"id":"https://openalex.org/C2779313563","wikidata":"https://www.wikidata.org/wiki/Q17072565","display_name":"On Language","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2605.22769","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2605.22769","pdf_url":"https://arxiv.org/pdf/2605.22769","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-05720736v1","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-05720736v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ICML, Jul 2026, S\u00e9oul, South Korea","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"doi:10.48550/arxiv.2605.22769","doi":"https://doi.org/10.48550/arxiv.2605.22769","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22769","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2605.22769","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2605.22769","pdf_url":"https://arxiv.org/pdf/2605.22769","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204383257.pdf","grobid_xml":"https://content.openalex.org/works/W7204383257.grobid-xml"},"referenced_works_count":6,"referenced_works":["https://openalex.org/W3177765786","https://openalex.org/W4411120158","https://openalex.org/W4414939355","https://openalex.org/W4415237786","https://openalex.org/W4416865542","https://openalex.org/W4417465430"],"related_works":["https://openalex.org/W3174266714","https://openalex.org/W4416353947","https://openalex.org/W4402670271","https://openalex.org/W4388717467","https://openalex.org/W4404782276","https://openalex.org/W4391987974","https://openalex.org/W4416865542","https://openalex.org/W4403755292","https://openalex.org/W4415797451","https://openalex.org/W4409628940"],"abstract_inverted_index":{"Large":[0],"language":[1,113],"models":[2,11,74,87,107],"(LLMs)":[3],"are":[4,51],"typically":[5],"trained":[6,106],"on":[7,36,45,88,111,138,170],"shuffled":[8,99,109,135],"corpora,":[9],"yielding":[10],"whose":[12,20],"knowledge":[13,117],"is":[14],"frozen":[15],"at":[16,156,162],"train":[17],"time":[18,81],"and":[19,65,94,115,123,160],"temporal":[21],"grounding":[22],"remains":[23],"poorly":[24],"understood.":[25],"In":[26],"this":[27],"work,":[28],"we":[29,54,84],"study":[30],"the":[31,37,151],"impact":[32],"of":[33,39,59,72,153],"pre-training":[34,129,136],"dynamics":[35],"acquisition":[38],"time-sensitive":[40],"factual":[41,132,145],"knowledge,":[42],"focusing":[43],"specifically":[44],"data":[46],"ordering.":[47],"Our":[48,101],"main":[49],"contributions":[50],"twofold.":[52],"First,":[53],"introduce":[55],"a":[56,165],"comprehensive":[57],"benchmark":[58],"over":[60],"7,000":[61],"temporally":[62,89,124],"grounded":[63],"questions":[64],"an":[66],"evaluation":[67],"protocol":[68],"that":[69,104],"enables":[70],"analysis":[71],"whether":[73],"correctly":[75],"associate":[76],"facts":[77],"with":[78,150],"their":[79],"corresponding":[80],"periods.":[82],"Second,":[83],"pretrain":[85],"6B-parameter":[86],"ordered":[90,128],"Common":[91],"Crawl":[92],"snapshots":[93],"compare":[95],"them":[96],"against":[97],"standard":[98],"pre-training.":[100],"results":[102],"show":[103],"sequentially":[105],"match":[108],"baselines":[110],"general":[112],"understanding":[114],"common":[116],"while":[118,134],"consistently":[119],"exhibiting":[120],"more":[121],"up-to-date":[122],"precise":[125],"knowledge.":[126],"Temporally":[127],"yields":[130],"improved":[131],"freshness,":[133],"peaks":[137],"older":[139],"data,":[140],"possibly":[141],"due":[142],"to":[143],"increased":[144],"repetition.":[146],"These":[147],"findings,":[148],"along":[149],"release":[152],"our":[154],"code":[155],"https://github.com/kyutai-labs/kairos":[157],",":[158],"checkpoints,":[159],"datasets":[161],"https://huggingface.co/collections/kyutai/kairos":[163],"provide":[164],"foundation":[166],"for":[167,173],"future":[168],"research":[169],"continual":[171],"learning":[172],"LLMs.":[174]},"counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-08-27T00:00:00"}