{"id":"https://openalex.org/W7134924016","doi":"https://doi.org/10.48550/arxiv.2603.09865","title":"GAST: Gradient-aligned Sparse Tuning of Large Language Models with Data-layer Selection","display_name":"GAST: Gradient-aligned Sparse Tuning of Large Language Models with Data-layer Selection","publication_year":2026,"publication_date":"2026-03-10","ids":{"openalex":"https://openalex.org/W7134924016","doi":"https://doi.org/10.48550/arxiv.2603.09865"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.09865","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09865","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.09865","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128793835","display_name":"Kai Yao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Kai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101337089","display_name":"Zhenghan Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Zhenghan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128701156","display_name":"Kaixin Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Kaixin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128793667","display_name":"Mingjie Zhong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhong, Mingjie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000351282","display_name":"Danzhao Cheng","orcid":"https://orcid.org/0000-0002-9303-7472"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Danzhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074276416","display_name":"Zhaorui Tan","orcid":"https://orcid.org/0000-0001-5054-8275"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tan, Zhaorui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128701970","display_name":"Yixin Ji","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ji, Yixin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100762977","display_name":"Peng Gao","orcid":"https://orcid.org/0000-0001-9868-2115"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Penglei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.24889999628067017,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.24889999628067017,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.1143999993801117,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.08309999853372574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.6244000196456909},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.6074000000953674},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5625},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.49079999327659607},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.47839999198913574},{"id":"https://openalex.org/keywords/feature-selection","display_name":"Feature selection","score":0.4122999906539917},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.40070000290870667}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7692000269889832},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.6244000196456909},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.6074000000953674},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5625},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5016000270843506},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4927000105381012},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.49079999327659607},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.47839999198913574},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.42590001225471497},{"id":"https://openalex.org/C148483581","wikidata":"https://www.wikidata.org/wiki/Q446488","display_name":"Feature selection","level":2,"score":0.4122999906539917},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.40070000290870667},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.3319000005722046},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.3303000032901764},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.3237999975681305},{"id":"https://openalex.org/C56372850","wikidata":"https://www.wikidata.org/wiki/Q1050404","display_name":"Sparse matrix","level":3,"score":0.31200000643730164},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.27570000290870667},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.2606000006198883},{"id":"https://openalex.org/C77637269","wikidata":"https://www.wikidata.org/wiki/Q7002051","display_name":"Neural coding","level":2,"score":0.2597000002861023}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.09865","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09865","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.09865","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09865","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Parameter-Efficient":[0],"Fine-Tuning":[1],"(PEFT)":[2],"has":[3],"become":[4],"a":[5,121,133,148,158,170],"key":[6,24],"strategy":[7,135],"for":[8,144,173],"adapting":[9],"large":[10],"language":[11],"models,":[12],"with":[13],"recent":[14],"advances":[15],"in":[16,129,176],"sparse":[17],"tuning":[18],"reducing":[19],"overhead":[20],"by":[21,131],"selectively":[22],"updating":[23],"parameters":[25],"or":[26],"subsets":[27,57],"of":[28,90,120],"data.":[29],"Existing":[30],"approaches":[31,155],"generally":[32],"focus":[33],"on":[34],"two":[35],"distinct":[36,76],"paradigms:":[37],"layer-selective":[38],"methods":[39,51,63],"aiming":[40,52],"to":[41,45,53,58,75,157],"fine-tune":[42],"critical":[43],"layers":[44],"minimize":[46],"computational":[47],"load,":[48],"and":[49,79,114,151],"data-selective":[50],"select":[54],"effective":[55],"training":[56],"boost":[59],"training.":[60],"However,":[61],"current":[62],"typically":[64],"overlook":[65],"the":[66,139],"fact":[67],"that":[68,106,136,163],"different":[69],"data":[70,87,113,142],"points":[71,143],"contribute":[72],"varying":[73],"degrees":[74],"model":[77],"layers,":[78],"they":[80],"often":[81],"discard":[82],"potentially":[83],"valuable":[84],"information":[85,130],"from":[86],"perceived":[88],"as":[89,117],"low":[91],"quality.":[92],"To":[93],"address":[94],"these":[95],"limitations,":[96],"we":[97],"propose":[98],"Gradient-aligned":[99],"Sparse":[100],"Tuning":[101],"(GAST),":[102],"an":[103],"innovative":[104],"method":[105],"simultaneously":[107],"performs":[108],"selective":[109],"fine-tuning":[110],"at":[111],"both":[112],"layer":[115],"dimensions":[116],"integral":[118],"components":[119],"unified":[122],"optimization":[123],"strategy.":[124],"GAST":[125,164],"specifically":[126],"targets":[127],"redundancy":[128],"employing":[132],"layer-sparse":[134],"adaptively":[137],"selects":[138],"most":[140],"impactful":[141],"each":[145],"layer,":[146],"providing":[147],"more":[149],"comprehensive":[150],"sophisticated":[152],"solution":[153],"than":[154],"restricted":[156],"single":[159],"dimension.":[160],"Experiments":[161],"demonstrate":[162],"consistently":[165],"outperforms":[166],"baseline":[167],"methods,":[168],"establishing":[169],"promising":[171],"direction":[172],"future":[174],"research":[175],"PEFT":[177],"strategies.":[178]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-12T00:00:00"}