{"id":"https://openalex.org/W4391621451","doi":"https://doi.org/10.48550/arxiv.2402.02834","title":"Shortened LLaMA: Depth Pruning for Large Language Models with Comparison of Retraining Methods","display_name":"Shortened LLaMA: Depth Pruning for Large Language Models with Comparison of Retraining Methods","publication_year":2024,"publication_date":"2024-02-05","ids":{"openalex":"https://openalex.org/W4391621451","doi":"https://doi.org/10.48550/arxiv.2402.02834"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2402.02834","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.02834","pdf_url":"https://arxiv.org/pdf/2402.02834","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4391621451"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2402.02834","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103109289","display_name":"Bo-Kyeong Kim","orcid":"https://orcid.org/0000-0002-0224-7985","observed_orcids":["https://orcid.org/0000-0002-0224-7985"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Bo-Kyeong","raw_affiliation_strings":["Nota Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nota Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066858751","display_name":"Geonmin Kim","orcid":"https://orcid.org/0000-0002-9704-6843","observed_orcids":["https://orcid.org/0000-0002-9704-6843"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Geonmin","raw_affiliation_strings":["Nota Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nota Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100452223","display_name":"Taeho Kim","orcid":"https://orcid.org/0000-0003-4522-5388","observed_orcids":["https://orcid.org/0000-0003-4522-5388","https://orcid.org/0000-0001-9329-0614"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Tae-Ho","raw_affiliation_strings":["Nota Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nota Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037028526","display_name":"Thibault Castells","orcid":"https://orcid.org/0009-0004-0549-5695","observed_orcids":["https://orcid.org/0009-0004-0549-5695"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Castells, Thibault","raw_affiliation_strings":["Nota Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nota Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101784669","display_name":"Shinkook Choi","orcid":"https://orcid.org/0000-0002-9617-2418","observed_orcids":["https://orcid.org/0000-0002-9617-2418"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Choi, Shinkook","raw_affiliation_strings":["Nota Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nota Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101765002","display_name":"Junho Shin","orcid":"https://orcid.org/0000-0002-7008-0902","observed_orcids":["https://orcid.org/0000-0002-7008-0902"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shin, Junho","raw_affiliation_strings":["Nota Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nota Inc","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032291620","display_name":"Hyoung\u2010Kyu Song","orcid":"https://orcid.org/0000-0002-3274-4982","observed_orcids":["https://orcid.org/0000-0002-3274-4982"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Hyoung-Kyu","raw_affiliation_strings":["Nota Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nota Inc","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":151,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9726999998092651,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9726999998092651,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.019200000911951065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0017999999690800905,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.843999981880188},{"id":"https://openalex.org/keywords/structured-pruning","display_name":"structured pruning","score":0.8069999814033508},{"id":"https://openalex.org/keywords/llama","display_name":"llamas","score":0.6690000295639038},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.7440000176429749},{"id":"https://openalex.org/keywords/inference-efficiency","display_name":"inference efficiency","score":0.9110000133514404},{"id":"https://openalex.org/keywords/continual-pre-training","display_name":"continual pre-training","score":0.7549999952316284}],"concepts":[{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.791198194026947},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.64268958568573},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.47057029604911804},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3808450698852539},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3568022549152374},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.33117952942848206},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.32670408487319946},{"id":"https://openalex.org/C144027150","wikidata":"https://www.wikidata.org/wiki/Q48803","display_name":"Horticulture","level":1,"score":0.19583263993263245},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.18587404489517212},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.09237629175186157},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2402.02834","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.02834","pdf_url":"https://arxiv.org/pdf/2402.02834","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2402.02834","doi":"https://doi.org/10.48550/arxiv.2402.02834","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.02834","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2402.02834","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.02834","pdf_url":"https://arxiv.org/pdf/2402.02834","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320328359","display_name":"Ministry of Science and ICT, South Korea","ror":"https://ror.org/01wpjm123"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391621451.pdf","grobid_xml":"https://content.openalex.org/works/W4391621451.grobid-xml"},"referenced_works_count":16,"referenced_works":["https://openalex.org/W2794325560","https://openalex.org/W2996908057","https://openalex.org/W4224308101","https://openalex.org/W4229005866","https://openalex.org/W4308760226","https://openalex.org/W4319792274","https://openalex.org/W4322718191","https://openalex.org/W4327810158","https://openalex.org/W4378474066","https://openalex.org/W4383604511","https://openalex.org/W4384648639","https://openalex.org/W4389500833","https://openalex.org/W4391631305","https://openalex.org/W4393967974","https://openalex.org/W4404918643","https://openalex.org/W4412887764"],"related_works":["https://openalex.org/W7128361988","https://openalex.org/W7117125605","https://openalex.org/W7172021105","https://openalex.org/W7162637510","https://openalex.org/W4404781961","https://openalex.org/W7161739926","https://openalex.org/W4391899168","https://openalex.org/W4415795942","https://openalex.org/W4416043746","https://openalex.org/W7202113624"],"abstract_inverted_index":{"Structured":[0],"pruning":[1,20,98,112,115,134,157],"of":[2,13,24,37,52,67],"modern":[3],"large":[4,148],"language":[5],"models":[6,140,172],"(LLMs)":[7],"has":[8],"emerged":[9],"as":[10],"a":[11,65,147],"way":[12],"decreasing":[14],"their":[15,84],"high":[16],"computational":[17],"needs.":[18],"Width":[19],"reduces":[21],"the":[22,35,50,53,77],"size":[23,51],"projection":[25],"weight":[26],"matrices":[27],"(e.g.,":[28],"by":[29],"removing":[30],"attention":[31],"heads)":[32],"while":[33,48,103],"maintaining":[34],"number":[36],"layers.":[38],"Depth":[39],"pruning,":[40,71],"in":[41],"contrast,":[42],"removes":[43],"entire":[44],"layers":[45],"or":[46,64,106],"blocks,":[47],"keeping":[49],"remaining":[54],"weights":[55],"unchanged.":[56],"Most":[57],"current":[58],"research":[59],"focuses":[60],"on":[61,86,146],"either":[62],"width-only":[63],"blend":[66],"width":[68,111,133],"and":[69,171],"depth":[70,97],"with":[72],"little":[73],"comparative":[74],"analysis":[75],"between":[76],"two":[78],"units":[79],"(width":[80],"vs.":[81],"depth)":[82],"concerning":[83],"impact":[85],"LLM":[87],"inference":[88,118],"efficiency.":[89],"In":[90,137],"this":[91,161],"work,":[92],"we":[93],"show":[94],"that":[95,124],"simple":[96],"can":[99,163,173],"effectively":[100],"compress":[101],"LLMs":[102],"achieving":[104],"comparable":[105],"superior":[107],"performance":[108],"to":[109],"recent":[110],"studies.":[113],"Our":[114],"method":[116],"boosts":[117],"speeds,":[119],"especially":[120],"under":[121],"memory-constrained":[122],"conditions":[123],"require":[125],"limited":[126],"batch":[127],"sizes":[128],"for":[129,141],"running":[130],"LLMs,":[131],"where":[132],"is":[135],"ineffective.":[136],"retraining":[138],"pruned":[139],"quality":[142],"recovery,":[143],"continued":[144],"pretraining":[145],"corpus":[149],"markedly":[150],"outperforms":[151],"LoRA-based":[152],"tuning,":[153],"particularly":[154],"at":[155],"severe":[156],"ratios.":[158],"We":[159],"hope":[160],"work":[162],"help":[164],"build":[165],"compact":[166],"yet":[167],"capable":[168],"LLMs.":[169],"Code":[170],"be":[174],"found":[175],"at:":[176],"https://github.com/Nota-NetsPresso/shortened-llm":[177]},"counts_by_year":[{"year":2026,"cited_by_count":63},{"year":2025,"cited_by_count":57},{"year":2024,"cited_by_count":30},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-14T11:04:58.165337","created_date":"2024-02-08T00:00:00"}
