{"id":"https://openalex.org/W4416235437","doi":"https://doi.org/10.48550/arxiv.2511.08877","title":"Hierarchical Memorization in Large Language Models: Evidence from Citation Generation","display_name":"Hierarchical Memorization in Large Language Models: Evidence from Citation Generation","publication_year":2025,"publication_date":"2025-11-12","ids":{"openalex":"https://openalex.org/W4416235437","doi":"https://doi.org/10.48550/arxiv.2511.08877"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2511.08877","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2511.08877","pdf_url":"https://arxiv.org/pdf/2511.08877","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2511.08877","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029152172","display_name":"Junichiro Niimi","orcid":"https://orcid.org/0000-0002-4618-6272","observed_orcids":["https://orcid.org/0000-0002-4618-6272"]},"institutions":[{"id":"https://openalex.org/I4210110652","display_name":"RIKEN","ror":"https://ror.org/01sjwvz98","country_code":"JP","type":"facility","lineage":["https://openalex.org/I4210110652"]},{"id":"https://openalex.org/I4210126580","display_name":"RIKEN Center for Advanced Intelligence Project","ror":"https://ror.org/03ckxwf91","country_code":"JP","type":"facility","lineage":["https://openalex.org/I4210110652","https://openalex.org/I4210126580"]},{"id":"https://openalex.org/I96636082","display_name":"Meijo University","ror":"https://ror.org/04h42fc75","country_code":"JP","type":"education","lineage":["https://openalex.org/I96636082"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Niimi, Junichiro","raw_affiliation_strings":["Meijo University","RIKEN AIP"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meijo University","institution_ids":["https://openalex.org/I96636082"]},{"raw_affiliation_string":"RIKEN AIP","institution_ids":["https://openalex.org/I4210110652","https://openalex.org/I4210126580"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5029152172"],"corresponding_institution_ids":["https://openalex.org/I4210110652","https://openalex.org/I4210126580","https://openalex.org/I96636082"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9707000255584717,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9707000255584717,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.011300000362098217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.0020000000949949026,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8899999856948853},{"id":"https://openalex.org/keywords/memorization","display_name":"memorization","score":0.7960000038146973},{"id":"https://openalex.org/keywords/hallucinations","display_name":"hallucinations","score":0.8679999709129333},{"id":"https://openalex.org/keywords/citation-generation","display_name":"citation generation","score":0.9539999961853027},{"id":"https://openalex.org/keywords/bibliographic-records","display_name":"bibliographic records","score":0.6740000247955322},{"id":"https://openalex.org/keywords/factual-accuracy","display_name":"factual accuracy","score":0.6359999775886536},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.8420000076293945}],"concepts":[{"id":"https://openalex.org/C2911011789","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Hallucinating","level":2,"score":0.892799973487854},{"id":"https://openalex.org/C2778805511","wikidata":"https://www.wikidata.org/wiki/Q1713","display_name":"Citation","level":2,"score":0.617900013923645},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5979999899864197},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5702000260353088},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5170000195503235},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.48170000314712524},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.4180999994277954},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.4074999988079071},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.4000999927520752},{"id":"https://openalex.org/C2780695315","wikidata":"https://www.wikidata.org/wiki/Q3799040","display_name":"Unobservable","level":2,"score":0.3882000148296356},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.38530001044273376},{"id":"https://openalex.org/C134537474","wikidata":"https://www.wikidata.org/wiki/Q17144832","display_name":"Naturalness","level":2,"score":0.3824000060558319},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3743000030517578},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.3619000017642975},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3522999882698059},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.34869998693466187},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.34279999136924744},{"id":"https://openalex.org/C2779602731","wikidata":"https://www.wikidata.org/wiki/Q30067981","display_name":"Misrepresentation","level":2,"score":0.33550000190734863},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.33469998836517334},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.33329999446868896},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.3239000141620636},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.31619998812675476},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2973000109195709},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.28760001063346863},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.28349998593330383},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.2833000123500824},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.2563999891281128}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2511.08877","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2511.08877","pdf_url":"https://arxiv.org/pdf/2511.08877","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2511.08877","doi":"https://doi.org/10.48550/arxiv.2511.08877","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2511.08877","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2511.08877","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2511.08877","pdf_url":"https://arxiv.org/pdf/2511.08877","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416235437.pdf","grobid_xml":"https://content.openalex.org/works/W4416235437.grobid-xml"},"referenced_works_count":5,"referenced_works":["https://openalex.org/W2973379954","https://openalex.org/W4296932804","https://openalex.org/W4391158659","https://openalex.org/W4401306886","https://openalex.org/W7155201584"],"related_works":["https://openalex.org/W7102604627","https://openalex.org/W7162669254","https://openalex.org/W7133781660","https://openalex.org/W4411113477","https://openalex.org/W7134840272","https://openalex.org/W4404792993","https://openalex.org/W7154991901","https://openalex.org/W7155172492","https://openalex.org/W7133217935","https://openalex.org/W7223288327"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"generate":[4],"fluent":[5],"text":[6],"across":[7,78,99],"a":[8,21,50,65,121,187,192],"wide":[9],"range":[10],"of":[11,16,39,202],"tasks,":[12],"but":[13,191],"the":[14,40,108,199,205],"fabrication":[15],"non-existent":[17],"academic":[18],"citations":[19,77,119,126],"remains":[20],"critical":[22],"and":[23,34,56,73,101,120,140,147,154,160,172],"well-documented":[24],"failure":[25],"mode.":[26],"Building":[27],"on":[28],"prior":[29],"work":[30],"that":[31,93],"frames":[32],"hallucination":[33],"verbatim":[35],"memorization":[36,135],"as":[37,49,178],"outcomes":[38],"same":[41],"probabilistic":[42],"process,":[43],"this":[44,59],"study":[45],"uses":[46],"citation":[47,105],"count":[48],"proxy":[51],"for":[52],"training":[53],"data":[54],"redundancy":[55,60,153],"asks":[57],"how":[58],"is":[61,136,184],"internally":[62],"structured":[63],"within":[64],"single":[66],"bibliographic":[67],"record.":[68],"Using":[69],"GPT-4.1,":[70],"we":[71],"generated":[72],"manually":[74],"verified":[75],"100":[76],"twenty":[79],"computer-science":[80],"domains,":[81],"measuring":[82],"factual":[83,95],"fidelity":[84],"via":[85],"cosine":[86],"similarity":[87],"against":[88],"authentic":[89],"metadata.":[90],"We":[91],"find":[92],"(i)":[94],"accuracy":[96],"varies":[97],"substantially":[98],"domains":[100],"scales":[102],"log-linearly":[103],"with":[104,138],"count,":[106],"(ii)":[107],"model":[109],"crosses":[110],"two":[111],"empirically":[112],"identifiable":[113],"thresholds;":[114],"an":[115,175],"inflection":[116],"around":[117],"90":[118],"saturation":[122],"point":[123],"near":[124],"1,200":[125],"beyond":[127],"which":[128],"records":[129,165],"are":[130],"reproduced":[131],"nearly":[132],"verbatim,":[133],"(iii)":[134],"hierarchical,":[137],"titles":[139,171],"first":[141],"authors":[142,173],"recalled":[143],"earliest":[144],"while":[145],"venues":[146],"numeric":[148],"fields":[149],"require":[150],"far":[151],"greater":[152],"publication":[155],"years":[156],"remain":[157],"essentially":[158],"unlearned,":[159],"(iv)":[161],"even":[162],"highly":[163],"cited":[164],"can":[166],"be":[167],"conflated":[168],"when":[169],"their":[170],"overlap,":[174],"effect":[176],"interpretable":[177],"spurious-attractor":[179],"interference.":[180],"Memorization":[181],"in":[182,204],"LLMs":[183],"therefore":[185],"not":[186],"binary":[188],"on/off":[189],"state":[190],"graduated,":[193],"hierarchically":[194],"layered":[195],"phenomenon":[196],"shaped":[197],"by":[198],"uneven":[200],"distribution":[201],"knowledge":[203],"pretraining":[206],"corpus.":[207]},"counts_by_year":[{"year":2026,"cited_by_count":5}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-11-14T00:00:00"}