{"id":"https://openalex.org/W7102387716","doi":"https://doi.org/10.48550/arxiv.2510.24256","title":"From Memorization to Reasoning in the Spectrum of Loss Curvature","display_name":"From Memorization to Reasoning in the Spectrum of Loss Curvature","alternate_titles":[],"publication_year":2025,"publication_date":"2025-10-28","ids":{"openalex":"https://openalex.org/W7102387716","doi":"https://doi.org/10.48550/arxiv.2510.24256"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2510.24256","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.24256","pdf_url":"https://arxiv.org/pdf/2510.24256","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W7102387716"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.24256","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5062612448","display_name":"Jack Merullo","orcid":"https://orcid.org/0009-0005-9673-6809","observed_orcids":["https://orcid.org/0009-0005-9673-6809"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Merullo, Jack","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Vatsavaya, Srihita","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vatsavaya, Srihita","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032905139","display_name":"Lucius Bushnaq","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bushnaq, Lucius","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128239812","display_name":"Owen Lewis","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lewis, Owen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.23080000281333923,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.23080000281333923,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.21699999272823334,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.17350000143051147,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/memorization","display_name":"memorization","score":0.9319999814033508},{"id":"https://openalex.org/keywords/loss-landscape-curvature","display_name":"loss landscape curvature","score":0.9409999847412109},{"id":"https://openalex.org/keywords/unlearning","display_name":"unlearning","score":0.6179999709129333},{"id":"https://openalex.org/keywords/vision-transformer","display_name":"vision transformer","score":0.5929999947547913},{"id":"https://openalex.org/keywords/language-models","display_name":"language models","score":0.9779999852180481},{"id":"https://openalex.org/keywords/weight-decomposition","display_name":"weight decomposition","score":0.5379999876022339},{"id":"https://openalex.org/keywords/fact-retrieval","display_name":"fact retrieval","score":0.7129999995231628},{"id":"https://openalex.org/keywords/arithmetic-reasoning","display_name":"arithmetic reasoning","score":0.828000009059906},{"id":"https://openalex.org/keywords/general-reasoning","display_name":"general reasoning","score":0.41499999165534973}],"concepts":[{"id":"https://openalex.org/C30038468","wikidata":"https://www.wikidata.org/wiki/Q4354775","display_name":"Memorization","level":2,"score":0.77920001745224},{"id":"https://openalex.org/C195065555","wikidata":"https://www.wikidata.org/wiki/Q214881","display_name":"Curvature","level":2,"score":0.6189000010490417},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5626999735832214},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5580000281333923},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49790000915527344},{"id":"https://openalex.org/C131598440","wikidata":"https://www.wikidata.org/wiki/Q107342","display_name":"Syllogism","level":2,"score":0.4652000069618225},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4124999940395355},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4043999910354614},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3804999887943268},{"id":"https://openalex.org/C2777379011","wikidata":"https://www.wikidata.org/wiki/Q938545","display_name":"Implicit learning","level":3,"score":0.3222000002861023},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.3091000020503998},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3084999918937683},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3077999949455261},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.29499998688697815},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2937000095844269},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.28540000319480896},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.26019999384880066},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.25690001249313354},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.25589999556541443},{"id":"https://openalex.org/C21847791","wikidata":"https://www.wikidata.org/wiki/Q191081","display_name":"Logical conjunction","level":2,"score":0.2531999945640564},{"id":"https://openalex.org/C2985612853","wikidata":"https://www.wikidata.org/wiki/Q185816","display_name":"Analogical reasoning","level":3,"score":0.25220000743865967},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.2506999969482422}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2510.24256","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.24256","pdf_url":"https://arxiv.org/pdf/2510.24256","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.24256","doi":"https://doi.org/10.48550/arxiv.2510.24256","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.24256","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.24256","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.24256","pdf_url":"https://arxiv.org/pdf/2510.24256","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7102387716.pdf","grobid_xml":"https://content.openalex.org/works/W7102387716.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W2117539524","https://openalex.org/W2155894447","https://openalex.org/W2156150815","https://openalex.org/W2523060838","https://openalex.org/W2535690855","https://openalex.org/W2912713668","https://openalex.org/W2912811302","https://openalex.org/W3152884768","https://openalex.org/W4221159672","https://openalex.org/W4389520705","https://openalex.org/W4392575839","https://openalex.org/W4393399301","https://openalex.org/W4398191506","https://openalex.org/W4401041523","https://openalex.org/W4401042420","https://openalex.org/W4401042854","https://openalex.org/W4403122979","https://openalex.org/W4404334616","https://openalex.org/W4404569590","https://openalex.org/W4404792993","https://openalex.org/W4405095095","https://openalex.org/W4405419023","https://openalex.org/W4406032137","https://openalex.org/W4407124196","https://openalex.org/W4411119670","https://openalex.org/W4411120158","https://openalex.org/W4415181430","https://openalex.org/W4415320092","https://openalex.org/W4415332740","https://openalex.org/W4416036658","https://openalex.org/W4417078371","https://openalex.org/W4417089128","https://openalex.org/W7133236469","https://openalex.org/W7133239630"],"related_works":["https://openalex.org/W4414968839","https://openalex.org/W4412944313","https://openalex.org/W3107969673","https://openalex.org/W7214732045","https://openalex.org/W4403882442","https://openalex.org/W6982645102","https://openalex.org/W4386566901","https://openalex.org/W4400065270","https://openalex.org/W7161831637","https://openalex.org/W7196947412"],"abstract_inverted_index":{"We":[0,158,183],"characterize":[1],"how":[2],"memorization":[3,218],"is":[4,39,55,156],"represented":[5],"in":[6,16,116,131,167,206,219,236],"transformer":[7],"models":[8,22],"and":[9,24,44,133,138,142,152,203,228,241],"show":[10],"that":[11,48,83,135,199],"it":[12],"can":[13,70],"be":[14],"disentangled":[15],"the":[17,33,49,105,121,204,210,215],"weights":[18],"of":[19,88,107,176,217],"both":[20],"language":[21],"(LMs)":[23],"vision":[25],"transformers":[26],"(ViTs)":[27],"using":[28],"a":[29,72,79,95,110,188],"decomposition":[30],"based":[31,40],"on":[32,41,125,128,164],"loss":[34],"landscape":[35],"curvature.":[36],"This":[37,77],"insight":[38],"prior":[42],"theoretical":[43],"empirical":[45],"work":[46,213],"showing":[47,187],"curvature":[50,69,108,197],"for":[51,113,231],"memorized":[52,90],"training":[53],"points":[54],"much":[56],"sharper":[57],"than":[58,94,171],"non":[59],"memorized,":[60],"meaning":[61],"ordering":[62],"weight":[63,80,168],"components":[64,198],"from":[65],"high":[66],"to":[67],"low":[68,196],"reveal":[71],"distinction":[73],"without":[74],"explicit":[75],"labels.":[76],"motivates":[78],"editing":[81,122],"procedure":[82,123],"suppresses":[84],"far":[85],"more":[86,92],"recitation":[87],"untargeted":[89],"data":[91],"effectively":[93],"recent":[96],"unlearning":[97],"method":[98],"(BalancedSubnet),":[99],"while":[100],"maintaining":[101],"lower":[102],"perplexity.":[103],"Since":[104],"basis":[106],"has":[109],"natural":[111],"interpretation":[112],"shared":[114],"structure":[115],"model":[117],"weights,":[118],"we":[119,200],"analyze":[120],"extensively":[124],"its":[126],"effect":[127],"downstream":[129],"tasks":[130,161,238],"LMs,":[132],"find":[134],"fact":[136,150,242],"retrieval":[137,151],"arithmetic":[139],"are":[140,181],"specifically":[141],"consistently":[143],"negatively":[144],"affected,":[145],"even":[146],"though":[147],"open":[148],"book":[149],"general":[153,172],"logical":[154],"reasoning":[155],"conserved.":[157],"posit":[159],"these":[160],"rely":[162],"heavily":[163],"specialized":[165],"directions":[166],"space":[169],"rather":[170],"purpose":[173],"mechanisms,":[174],"regardless":[175],"whether":[177],"those":[178],"individual":[179],"datapoints":[180],"memorized.":[182],"support":[184],"this":[185],"by":[186],"correspondence":[189],"between":[190],"task":[191,207],"data's":[192],"activation":[193],"strength":[194],"with":[195,222],"edit":[201],"out,":[202],"drop":[205],"performance":[208],"after":[209],"edit.":[211],"Our":[212],"enhances":[214],"understanding":[216],"neural":[220],"networks":[221],"practical":[223],"applications":[224],"towards":[225],"removing":[226],"it,":[227],"provides":[229],"evidence":[230],"idiosyncratic,":[232],"narrowly-used":[233],"structures":[234],"involved":[235],"solving":[237],"like":[239],"math":[240],"retrieval.":[243]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7102387716","counts_by_year":[{"year":2026,"cited_by_count":4}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2025-10-30T00:00:00"}
