{"id":"https://openalex.org/W7162652628","doi":"https://doi.org/10.48550/arxiv.2605.27774","title":"Fine-Tuning Dynamics of In-Context Factual Recall in Transformers","display_name":"Fine-Tuning Dynamics of In-Context Factual Recall in Transformers","publication_year":2026,"publication_date":"2026-05-26","ids":{"openalex":"https://openalex.org/W7162652628","doi":"https://doi.org/10.48550/arxiv.2605.27774"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.27774","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27774","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.27774","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137306265","display_name":"Ruomin Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Ruomin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004602170","display_name":"Eshaan Nichani","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nichani, Eshaan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137234827","display_name":"Jason D. Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Jason D.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5035001911","display_name":"Rong Ge","orcid":"https://orcid.org/0000-0002-2218-3675"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ge, Rong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.47699999809265137,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.47699999809265137,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.1446000039577484,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.030400000512599945,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.7285000085830688},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.7261000275611877},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.5401999950408936},{"id":"https://openalex.org/keywords/associative-property","display_name":"Associative property","score":0.4212000072002411},{"id":"https://openalex.org/keywords/precision-and-recall","display_name":"Precision and recall","score":0.3553999960422516},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.34200000762939453},{"id":"https://openalex.org/keywords/supervised-learning","display_name":"Supervised learning","score":0.34060001373291016},{"id":"https://openalex.org/keywords/context-model","display_name":"Context model","score":0.3395000100135803},{"id":"https://openalex.org/keywords/dynamics","display_name":"Dynamics (music)","score":0.3366999924182892}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7584999799728394},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.7285000085830688},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.7261000275611877},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5913000106811523},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.5401999950408936},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48159998655319214},{"id":"https://openalex.org/C159423971","wikidata":"https://www.wikidata.org/wiki/Q177251","display_name":"Associative property","level":2,"score":0.4212000072002411},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3792000114917755},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.3553999960422516},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.34200000762939453},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.34060001373291016},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.3395000100135803},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.3366999924182892},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.31150001287460327},{"id":"https://openalex.org/C120567893","wikidata":"https://www.wikidata.org/wiki/Q1582085","display_name":"Knowledge extraction","level":2,"score":0.30379998683929443},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.29989999532699585},{"id":"https://openalex.org/C49929091","wikidata":"https://www.wikidata.org/wiki/Q1930471","display_name":"General knowledge","level":2,"score":0.28949999809265137},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.28769999742507935},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.28209999203681946},{"id":"https://openalex.org/C24138899","wikidata":"https://www.wikidata.org/wiki/Q17141258","display_name":"Instance-based learning","level":3,"score":0.27709999680519104},{"id":"https://openalex.org/C12186640","wikidata":"https://www.wikidata.org/wiki/Q6815743","display_name":"Memory model","level":3,"score":0.2660999894142151},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2655999958515167},{"id":"https://openalex.org/C8038995","wikidata":"https://www.wikidata.org/wiki/Q1152135","display_name":"Unsupervised learning","level":2,"score":0.26510000228881836},{"id":"https://openalex.org/C40506919","wikidata":"https://www.wikidata.org/wiki/Q7452469","display_name":"Sequence learning","level":2,"score":0.2590999901294708},{"id":"https://openalex.org/C2983526489","wikidata":"https://www.wikidata.org/wiki/Q133500","display_name":"Associative learning","level":2,"score":0.25870001316070557},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.2572999894618988},{"id":"https://openalex.org/C53442348","wikidata":"https://www.wikidata.org/wiki/Q745101","display_name":"Content-addressable memory","level":3,"score":0.25690001249313354}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.27774","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27774","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.27774","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27774","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"In-context":[0],"learning":[1,45,58,88],"\\":[2,13,198],"--":[3,14,199],"performing":[4],"tasks":[5],"based":[6],"on":[7,41,169],"examples":[8],"given":[9],"in":[10,22,31,67,202],"the":[11,44,50,60,68,72,99,135,143,161,175,203,217,224],"prompt":[12],"is":[15,79,108,140,227],"an":[16],"important":[17],"capability":[18],"that":[19,174,216],"has":[20,27],"emerged":[21],"large":[23],"language":[24],"models":[25],"and":[26,34,126,133,172,214],"received":[28],"significant":[29],"attention":[30,186,219],"both":[32,128],"theory":[33],"practice.":[35],"Existing":[36],"theoretical":[37,212],"work":[38],"often":[39],"focuses":[40],"settings":[42],"where":[43,105],"uses":[46],"information":[47],"purely":[48],"from":[49,117],"prompt.":[51],"However,":[52],"many":[53],"practical":[54],"instances":[55],"of":[56,112,165,196,205,230],"in-context":[57,87],"require":[59],"model":[61,176],"to":[62,75,147,182],"retrieve":[63,134],"factual":[64,90,101],"knowledge":[65,78,91,139,207],"stored":[66,206],"model's":[69],"parameters,":[70],"with":[71,122],"context":[73,111],"serving":[74],"identify":[76],"which":[77],"relevant.":[80],"In":[81],"this":[82,95,130],"work,":[83],"we":[84],"study":[85],"how":[86],"leverages":[89],"recall.":[92],"We":[93,159],"formalize":[94],"behavior":[96],"by":[97,142,180],"introducing":[98],"\\emph{in-context":[100],"recall":[102],"(IC-recall)}":[103],"task,":[104],"a":[106,110,118,123,148,166,183,192],"transformer":[107,144,168],"provided":[109],"(subject,":[113,155],"answer)":[114,157],"pairs":[115],"generated":[116],"hidden":[119,131],"relation,":[120,156],"along":[121],"query":[124],"subject,":[125],"must":[127],"infer":[129],"relation":[132],"corresponding":[136],"answer.":[137],"Factual":[138],"modeled":[141],"having":[145],"access":[146],"simple":[149],"pre-constructed":[150],"MLP":[151,225],"associative":[152],"memory":[153],"storing":[154],"triplets.":[158,208],"analyze":[160],"supervised":[162],"fine-tuning":[163,189],"dynamics":[164],"one-layer":[167],"IC-recall":[170,179],"data":[171],"prove":[173],"successfully":[177],"performs":[178],"converging":[181],"particular":[184],"pairwise":[185,218],"pattern.":[187],"This":[188],"stage":[190],"requires":[191],"very":[193],"small":[194],"number":[195,204],"samples":[197],"only":[200],"polylogarithmic":[201],"Experiments":[209],"verify":[210],"our":[211],"predictions":[213],"show":[215],"pattern":[220],"emerges":[221],"even":[222],"when":[223],"layer":[226],"pretrained":[228],"instead":[229],"constructed.":[231]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-29T00:00:00"}