{"id":"https://openalex.org/W7137875768","doi":"https://doi.org/10.48550/arxiv.2603.15389","title":"When Does Sparsity Mitigate the Curse of Depth in LLMs","display_name":"When Does Sparsity Mitigate the Curse of Depth in LLMs","publication_year":2026,"publication_date":"2026-03-16","ids":{"openalex":"https://openalex.org/W7137875768","doi":"https://doi.org/10.48550/arxiv.2603.15389"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.15389","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.15389","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.15389","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033678891","display_name":"Dilxat Muhtar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Muhtar, Dilxat","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129689620","display_name":"Xinyuan Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Xinyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129741753","display_name":"Sebastian Pokutta","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pokutta, Sebastian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129645492","display_name":"Max Zimmer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zimmer, Max","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129681674","display_name":"Nico Pelleriti","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pelleriti, Nico","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129743827","display_name":"Thomas Hofmann","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hofmann, Thomas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129700652","display_name":"Shiwei Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Shiwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.27730000019073486,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.27730000019073486,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.19009999930858612,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.08020000159740448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.7303000092506409},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6692000031471252},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5940999984741211},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.5932000279426575},{"id":"https://openalex.org/keywords/curse","display_name":"Curse","score":0.5892999768257141},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5254999995231628},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.4837000072002411},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.45559999346733093}],"concepts":[{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.7303000092506409},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6692000031471252},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5940999984741211},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5932000279426575},{"id":"https://openalex.org/C2780273121","wikidata":"https://www.wikidata.org/wiki/Q109411","display_name":"Curse","level":2,"score":0.5892999768257141},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5699999928474426},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5254999995231628},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.4837000072002411},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.47620001435279846},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.45559999346733093},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43140000104904175},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3752000033855438},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3691999912261963},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.35199999809265137},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.3393000066280365},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3246000111103058},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.3163999915122986},{"id":"https://openalex.org/C6929976","wikidata":"https://www.wikidata.org/wiki/Q3771881","display_name":"Regulator","level":3,"score":0.29019999504089355},{"id":"https://openalex.org/C30772137","wikidata":"https://www.wikidata.org/wiki/Q5164762","display_name":"Consumption (sociology)","level":2,"score":0.2833999991416931},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2815999984741211},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.28139999508857727},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.27549999952316284},{"id":"https://openalex.org/C111030470","wikidata":"https://www.wikidata.org/wiki/Q1430460","display_name":"Curse of dimensionality","level":2,"score":0.27059999108314514},{"id":"https://openalex.org/C101097943","wikidata":"https://www.wikidata.org/wiki/Q5176983","display_name":"Counterintuitive","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2554999887943268},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.25380000472068787}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.15389","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.15389","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.15389","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.15389","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.5785886645317078}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"work":[1],"has":[2],"demonstrated":[3],"the":[4,30],"curse":[5],"of":[6,33,71],"depth":[7,64,193],"in":[8,35,109,115,191],"large":[9],"language":[10],"models":[11],"(LLMs),":[12],"where":[13],"later":[14],"layers":[15],"contribute":[16],"less":[17],"to":[18,29,145],"learning":[19],"and":[20,59,80,90,97,112,126,150,187],"representation":[21],"than":[22],"earlier":[23],"layers.":[24],"Such":[25],"under-utilization":[26],"is":[27,102,119,198],"linked":[28],"accumulated":[31],"growth":[32],"variance":[34,57,149],"Pre-Layer":[36],"Normalization,":[37],"which":[38,76,101],"can":[39,55],"push":[40],"deep":[41],"blocks":[42],"toward":[43],"near-identity":[44],"behavior.":[45],"In":[46],"this":[47],"paper,":[48],"we":[49,133],"provide":[50],"evidence":[51],"that":[52,180],"sparsity-like":[53,181],"mechanisms":[54,138],"dampen":[56],"propagation":[58],"are":[60,184],"associated":[61],"with":[62,139],"improved":[63],"utilization":[65],"Our":[66,117,177],"investigation":[67],"covers":[68],"two":[69],"sources":[70],"sparsity:":[72],"(i)":[73],"implicit":[74],"sparsity,":[75,100],"emerges":[77],"from":[78],"training":[79,165],"data":[81],"conditions,":[82],"including":[83,107],"weight":[84,88],"sparsity":[85,92,114],"induced":[86,93],"by":[87,94,104,122],"decay":[89],"attention":[91],"long-context":[95],"inputs;":[96],"(ii)":[98],"explicit":[99],"enforced":[103],"architectural":[105],"design,":[106],"key/value-sharing":[108],"Grouped-Query":[110],"Attention":[111],"expert-activation":[113],"Mixtureof-Experts.":[116],"claim":[118],"thoroughly":[120],"supported":[121],"controlled":[123],"depth-scaling":[124],"experiments":[125],"targeted":[127],"layer":[128,152],"effectiveness":[129],"interventions.":[130],"Across":[131],"settings,":[132],"observe":[134],"a":[135,160,169],"consistent":[136],"relationship:":[137],"reduced":[140],"effective":[141,192],"interaction":[142],"density":[143],"tend":[144],"exhibit":[146],"lower":[147],"output":[148],"better":[151],"differentiation.":[153],"We":[154],"eventually":[155],"distill":[156],"our":[157],"findings":[158],"into":[159],"practical":[161],"rule-of-thumb":[162],"recipe":[163],"for":[164,195],"depth-effective":[166],"LLMs,":[167],"yielding":[168],"notable":[170],"4.6":[171],"accuracy":[172],"improvement":[173],"on":[174],"downstream":[175],"tasks.":[176],"results":[178],"suggest":[179],"design":[182],"choices":[183],"an":[185],"important":[186],"previously":[188],"underemphasized":[189],"factor":[190],"scaling":[194],"LLMs.":[196],"Code":[197],"available":[199],"at":[200],"https://github.":[201],"com/pUmpKin-Co/SparsityAndCoD.":[202]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-18T00:00:00"}