{"id":"https://openalex.org/W7196947412","doi":"https://doi.org/10.52202/085713-3473","title":"For Better or for Worse, Transformers Seek Patterns for Memorization","display_name":"For Better or for Worse, Transformers Seek Patterns for Memorization","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W7196947412","doi":"https://doi.org/10.52202/085713-3473"},"language":"en","primary_location":{"id":"doi:10.52202/085713-3473","doi":"https://doi.org/10.52202/085713-3473","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-3473","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://infoscience.epfl.ch/handle/20.500.14299/266577","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014668625","display_name":"Madhur Panwar","orcid":"https://orcid.org/0000-0002-0053-733X","observed_orcids":["https://orcid.org/0000-0002-0053-733X"]},"institutions":[{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Madhur Panwar","raw_affiliation_strings":["EPFL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"EPFL","institution_ids":["https://openalex.org/I5124864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5157584183","display_name":"Gail Weiss","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Gail Weiss","raw_affiliation_strings":["Technion"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technion","institution_ids":["https://openalex.org/I174306211"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094344311","display_name":"Navin Goyal","orcid":"https://orcid.org/0009-0002-4795-3661","observed_orcids":["https://orcid.org/0009-0002-4795-3661"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Navin Goyal","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5088410008","display_name":"Antoine Bosselut","orcid":"https://orcid.org/0000-0001-8968-9649","observed_orcids":["https://orcid.org/0000-0001-8968-9649"]},"institutions":[{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Antoine Bosselut","raw_affiliation_strings":["EPFL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"EPFL","institution_ids":["https://openalex.org/I5124864"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.6375707,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"115103","last_page":"115133"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7271000146865845,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7271000146865845,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.07069999724626541,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.03420000150799751,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.8690000176429749},{"id":"https://openalex.org/keywords/memorization","display_name":"memorization","score":0.8989999890327454},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"pattern recognition","score":0.6449999809265137}],"concepts":[{"id":"https://openalex.org/C30038468","wikidata":"https://www.wikidata.org/wiki/Q4354775","display_name":"Memorization","level":2,"score":0.579800009727478},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5623999834060669},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.45969998836517334},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30329999327659607},{"id":"https://openalex.org/C2778833722","wikidata":"https://www.wikidata.org/wiki/Q28457372","display_name":"Tact","level":2,"score":0.29989999532699585},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.28279998898506165},{"id":"https://openalex.org/C101382669","wikidata":"https://www.wikidata.org/wiki/Q1616769","display_name":"Ferroresonance in electricity networks","level":4,"score":0.2409999966621399},{"id":"https://openalex.org/C2779606619","wikidata":"https://www.wikidata.org/wiki/Q17092524","display_name":"Interchangeability","level":2,"score":0.1979999989271164},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.18400000035762787},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.1808999925851822}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.52202/085713-3473","doi":"https://doi.org/10.52202/085713-3473","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-3473","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},{"id":"pmh:oai:infoscience.epfl.ch:20.500.14299/266577","doi":null,"is_oa":true,"landing_page_url":"https://infoscience.epfl.ch/handle/20.500.14299/266577","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I5124864","host_organization_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","host_organization_lineage":["https://openalex.org/I5124864"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference proceedings"}],"best_oa_location":{"id":"pmh:oai:infoscience.epfl.ch:20.500.14299/266577","doi":null,"is_oa":true,"landing_page_url":"https://infoscience.epfl.ch/handle/20.500.14299/266577","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I5124864","host_organization_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","host_organization_lineage":["https://openalex.org/I5124864"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference proceedings"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7214732045","https://openalex.org/W7133262701","https://openalex.org/W4386566901","https://openalex.org/W4400065270","https://openalex.org/W4404791712","https://openalex.org/W4417258277","https://openalex.org/W7140161910","https://openalex.org/W7125173877","https://openalex.org/W4412944313","https://openalex.org/W7166838861"],"abstract_inverted_index":{"Memorization":[0],"in":[1,17,68,78,100,125],"language":[2,19],"models":[3,20,74,133],"is":[4,35,85,160],"a":[5,37,101,141,156,170],"critical":[6],"yet":[7],"poorly":[8],"understood":[9],"phenomenon.":[10],"In":[11],"this":[12],"work,":[13],"we":[14,71,143],"investigate":[15],"memorization":[16,24,34,57,66,136],"transformer-based":[18],"by":[21,45,62,155],"analyzing":[22],"their":[23],"dynamics":[25,67],"during":[26,174],"training":[27],"over":[28,158],"multiple":[29],"epochs.":[30],"We":[31,104],"find":[32,106],"that":[33,73,82,92,107,149],"neither":[36],"constant":[38],"accumulation":[39],"of":[40,48,152],"sequences":[41,153],"nor":[42],"simply":[43],"dictated":[44],"the":[46,93,150,167],"recency":[47],"exposure":[49],"to":[50,59,96,112,138],"these":[51,98],"sequences.":[52],"Instead,":[53],"much":[54],"like":[55],"generalization,":[56],"appears":[58],"be":[60],"driven":[61],"pattern":[63],"recognition.":[64],"Tracking":[65],"mixed":[69],"datasets,":[70],"observe":[72],"memorize":[75],"different":[76],"sub-datasets":[77],"distinct":[79],"bursts,":[80],"suggesting":[81],"each":[83],"subset":[84,151],"associated":[86],"with":[87,127],"unique":[88],"underlying":[89],"patterns,":[90,131],"and":[91,163],"model":[94,157,171],"prefers":[95],"learn":[97],"patterns":[99,110,121],"consistent":[102],"order.":[103],"also":[105],"easily":[108],"learnable":[109],"tend":[111],"support":[113],"generalization":[114],"on":[115],"unseen":[116],"data,":[117],"while":[118],"more":[119],"complex":[120],"do":[122],"not.":[123],"Furthermore,":[124],"datasets":[126],"weak":[128],"or":[129],"absent":[130],"larger":[132],"may":[134],"delay":[135],"relative":[137],"smaller":[139],"ones,":[140],"behavior":[142],"term":[144],"overthinking.":[145],"Our":[146],"results":[147],"show":[148],"memorized":[154],"time":[159],"not":[161],"arbitrary,":[162],"give":[164],"insights":[165],"into":[166],"internal":[168],"processes":[169],"goes":[172],"through":[173],"training.":[175]},"counts_by_year":[],"updated_date":"2026-10-22T09:42:04.148104","created_date":"2026-08-07T00:00:00"}