{"id":"https://openalex.org/W7140309010","doi":"https://doi.org/10.48550/arxiv.2603.22355","title":"Demystifying Low-Rank Knowledge Distillation in Large Language Models: Convergence, Generalization, and Information-Theoretic Guarantees","display_name":"Demystifying Low-Rank Knowledge Distillation in Large Language Models: Convergence, Generalization, and Information-Theoretic Guarantees","publication_year":2026,"publication_date":"2026-03-22","ids":{"openalex":"https://openalex.org/W7140309010","doi":"https://doi.org/10.48550/arxiv.2603.22355"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.22355","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22355","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.22355","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5122971199","display_name":"Alberlucia Rafael Soarez","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Soarez, Alberlucia Rafael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130549388","display_name":"Daniel K. Kim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102724540","display_name":"Mariana Costa","orcid":"https://orcid.org/0000-0001-8195-1825"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Costa, Mariana","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5130606649","display_name":"Alejandro Torre","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Torre, Alejandro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1695999950170517,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1695999950170517,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.11550000309944153,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.07980000227689743,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.7967000007629395},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.713100016117096},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.7052000164985657},{"id":"https://openalex.org/keywords/projection","display_name":"Projection (relational algebra)","score":0.49810001254081726},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.4537999927997589},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.36640000343322754}],"concepts":[{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.7967000007629395},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.713100016117096},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.70660001039505},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.7052000164985657},{"id":"https://openalex.org/C57493831","wikidata":"https://www.wikidata.org/wiki/Q3134666","display_name":"Projection (relational algebra)","level":2,"score":0.49810001254081726},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.4537999927997589},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4397999942302704},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41269999742507935},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.36880001425743103},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.36640000343322754},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.35249999165534973},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.35179999470710754},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.2994000017642975},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.28369998931884766},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.2587999999523163},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.25130000710487366}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.22355","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22355","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.22355","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22355","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.5232945680618286,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Knowledge":[0],"distillation":[1,45,76],"has":[2],"emerged":[3],"as":[4,123],"a":[5,69],"powerful":[6],"technique":[7],"for":[8,73,157],"compressing":[9],"large":[10],"language":[11,78,178],"models":[12],"(LLMs)":[13],"into":[14],"efficient,":[15],"deployable":[16],"architectures":[17],"while":[18],"preserving":[19],"their":[20],"advanced":[21],"capabilities.":[22],"Recent":[23],"advances":[24],"in":[25,77,139],"low-rank":[26,74,86],"knowledge":[27,75],"distillation,":[28],"particularly":[29],"methods":[30,60],"like":[31],"Low-Rank":[32],"Clone":[33],"(LRC),":[34],"have":[35],"demonstrated":[36],"remarkable":[37],"empirical":[38,188],"success,":[39],"achieving":[40],"comparable":[41],"performance":[42],"to":[43],"full-parameter":[44],"with":[46,119,197],"significantly":[47],"reduced":[48],"training":[49],"data":[50],"and":[51,110,147,192],"computational":[52],"overhead.":[53],"However,":[54],"the":[55,89,104,115,120,132,141,145,171,187],"theoretical":[56,71,152,183],"foundations":[57],"underlying":[58],"these":[59],"remain":[61],"poorly":[62],"understood.":[63],"In":[64],"this":[65],"paper,":[66],"we":[67,126],"establish":[68],"rigorous":[70],"framework":[72],"models.":[79],"We":[80,98],"prove":[81],"that":[82,102,114,186],"under":[83],"mild":[84],"assumptions,":[85],"projection":[87],"preserves":[88],"optimization":[90],"dynamics,":[91],"yielding":[92],"explicit":[93],"convergence":[94],"rates":[95],"of":[96,131],"$O(1/\\sqrt{T})$.":[97],"derive":[99],"generalization":[100,111,116,193],"bounds":[101],"characterize":[103],"fundamental":[105],"trade-off":[106],"between":[107,144],"model":[108],"compression":[109],"capability,":[112],"showing":[113],"error":[117],"scales":[118],"rank":[121,158,164,190],"parameter":[122],"$O(r(m+n)/\\sqrt{n})$.":[124],"Furthermore,":[125],"provide":[127],"an":[128,162],"information-theoretic":[129],"analysis":[130],"activation":[133],"cloning":[134],"mechanism,":[135],"revealing":[136],"its":[137],"role":[138],"maximizing":[140],"mutual":[142],"information":[143],"teacher's":[146],"student's":[148],"intermediate":[149],"representations.":[150],"Our":[151],"results":[153],"offer":[154],"principled":[155],"guidelines":[156],"selection,":[159],"mathematically":[160],"suggesting":[161],"optimal":[163],"$r^*":[165],"=":[166],"O(\\sqrt{n})$":[167],"where":[168],"$n$":[169],"is":[170],"sample":[172],"size.":[173],"Experimental":[174],"validation":[175],"on":[176],"standard":[177],"modeling":[179],"benchmarks":[180],"confirms":[181],"our":[182,198],"predictions,":[184],"demonstrating":[185],"convergence,":[189],"scaling,":[191],"behaviors":[194],"align":[195],"closely":[196],"bounds.":[199]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-03-26T00:00:00"}