{"id":"https://openalex.org/W4406031123","doi":"https://doi.org/10.48550/arxiv.2501.00339","title":"GRASP: Replace Redundant Layers with Adaptive Singular Parameters for Efficient Model Compression","display_name":"GRASP: Replace Redundant Layers with Adaptive Singular Parameters for Efficient Model Compression","publication_year":2024,"publication_date":"2024-12-31","ids":{"openalex":"https://openalex.org/W4406031123","doi":"https://doi.org/10.48550/arxiv.2501.00339"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2501.00339","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.00339","pdf_url":"https://arxiv.org/pdf/2501.00339","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2501.00339","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016104023","display_name":"Kainan Liu","orcid":"https://orcid.org/0009-0005-6005-3619"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Liu, Kainan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100619206","display_name":"Yong Zhang","orcid":"https://orcid.org/0000-0003-1436-5951"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065576761","display_name":"Ning Cheng","orcid":"https://orcid.org/0000-0003-0909-5328"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Ning","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101511899","display_name":"Zhitao Li","orcid":"https://orcid.org/0000-0001-9967-1303"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Zhitao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100702012","display_name":"Shaojun Wang","orcid":"https://orcid.org/0000-0002-3680-9464"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Shaojun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5016038454","display_name":"Jing Xiao","orcid":"https://orcid.org/0000-0001-9615-4749"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Jing","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5016104023"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.8722000122070312,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.8722000122070312,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10904","display_name":"Embedded Systems Design Techniques","score":0.7943000197410583,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/singular-value-decomposition","display_name":"Singular value decomposition","score":0.6561211347579956},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6501898765563965},{"id":"https://openalex.org/keywords/decomposition","display_name":"Decomposition","score":0.6477944850921631},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.6194000244140625},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.5526939034461975},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.41602814197540283},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.3183251619338989},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.2120659351348877},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.18880656361579895},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1500760018825531},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.12381547689437866},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.11485493183135986},{"id":"https://openalex.org/keywords/composite-material","display_name":"Composite material","score":0.09668374061584473},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.06924569606781006}],"concepts":[{"id":"https://openalex.org/C22789450","wikidata":"https://www.wikidata.org/wiki/Q420904","display_name":"Singular value decomposition","level":2,"score":0.6561211347579956},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6501898765563965},{"id":"https://openalex.org/C124681953","wikidata":"https://www.wikidata.org/wiki/Q339062","display_name":"Decomposition","level":2,"score":0.6477944850921631},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.6194000244140625},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.5526939034461975},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.41602814197540283},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3183251619338989},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.2120659351348877},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18880656361579895},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1500760018825531},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.12381547689437866},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.11485493183135986},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.09668374061584473},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.06924569606781006},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2501.00339","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.00339","pdf_url":"https://arxiv.org/pdf/2501.00339","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2501.00339","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2501.00339","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2501.00339","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.00339","pdf_url":"https://arxiv.org/pdf/2501.00339","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4406031123.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3121932492","https://openalex.org/W4232638561","https://openalex.org/W1997544008","https://openalex.org/W2954752069","https://openalex.org/W1607100495","https://openalex.org/W3004137470","https://openalex.org/W131378092","https://openalex.org/W2938523900","https://openalex.org/W2366903352","https://openalex.org/W2117216852"],"abstract_inverted_index":{"Recent":[0],"studies":[1],"have":[2],"demonstrated":[3],"that":[4,57,114],"many":[5],"layers":[6,21,90],"are":[7],"functionally":[8],"redundant":[9,89],"in":[10,37],"large":[11],"language":[12],"models":[13],"(LLMs),":[14],"enabling":[15],"model":[16],"compression":[17,55,101,119,130],"by":[18,61],"removing":[19],"these":[20],"to":[22,79],"reduce":[23],"inference":[24],"cost.":[25],"While":[26],"such":[27],"approaches":[28],"can":[29],"improve":[30],"efficiency,":[31],"indiscriminate":[32],"layer":[33,68],"pruning":[34],"often":[35],"results":[36],"significant":[38],"performance":[39,105,127],"degradation.":[40],"In":[41],"this":[42,59],"paper,":[43],"we":[44],"propose":[45],"GRASP":[46,70,98,115],"(Gradient-based":[47],"Retention":[48],"of":[49,96,123],"Adaptive":[50],"Singular":[51],"Parameters),":[52],"a":[53,75,93],"novel":[54],"framework":[56],"mitigates":[58],"issue":[60],"preserving":[62],"sensitivity-aware":[63],"singular":[64,85],"values.":[65],"Unlike":[66],"direct":[67],"pruning,":[69],"leverages":[71],"gradient-based":[72],"attribution":[73],"on":[74],"small":[76],"calibration":[77],"dataset":[78],"adaptively":[80],"identify":[81],"and":[82],"retain":[83],"critical":[84],"components.":[86],"By":[87],"replacing":[88],"with":[91,106],"only":[92],"minimal":[94,107],"set":[95],"parameters,":[97],"achieves":[99],"efficient":[100],"while":[102],"maintaining":[103],"strong":[104],"overhead.":[108],"Experiments":[109],"across":[110],"multiple":[111],"LLMs":[112],"show":[113],"consistently":[116],"outperforms":[117],"existing":[118],"methods,":[120],"achieving":[121],"90%":[122],"the":[124],"original":[125],"model's":[126],"under":[128],"20%":[129],"ratio.":[131]},"counts_by_year":[],"updated_date":"2026-03-13T16:22:10.518609","created_date":"2025-01-04T00:00:00"}