{"id":"https://openalex.org/W4404411121","doi":"https://doi.org/10.1109/access.2024.3498904","title":"ELO-Mask: Effective and Layerwise Optimization of Mask for Sparse LLMs","display_name":"ELO-Mask: Effective and Layerwise Optimization of Mask for Sparse LLMs","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4404411121","doi":"https://doi.org/10.1109/access.2024.3498904"},"language":"en","primary_location":{"id":"doi:10.1109/access.2024.3498904","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2024.3498904","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/access.2024.3498904","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101201597","display_name":"Bingjie Xiang","orcid":null},"institutions":[{"id":"https://openalex.org/I2800556661","display_name":"China Tobacco","ror":"https://ror.org/030d08e08","country_code":"CN","type":"government","lineage":["https://openalex.org/I2800556661"]},{"id":"https://openalex.org/I4210118162","display_name":"Fujian Tobacco Industry Limited Liability Company (China)","ror":"https://ror.org/02fa7s823","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210118162"]},{"id":"https://openalex.org/I4210166335","display_name":"Xiamen Tobacco Industry (China)","ror":"https://ror.org/05t1nkw30","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210166335"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Bingjie Xiang","raw_affiliation_strings":["Information Center, China Tobacco Fujian Industrial Company Ltd., Xiamen, Fujian, China","Information Center, China Tobacco Fujian Industrial Co., Ltd, Xiamen, Fujian, China"],"affiliations":[{"raw_affiliation_string":"Information Center, China Tobacco Fujian Industrial Company Ltd., Xiamen, Fujian, China","institution_ids":["https://openalex.org/I4210118162","https://openalex.org/I4210166335"]},{"raw_affiliation_string":"Information Center, China Tobacco Fujian Industrial Co., Ltd, Xiamen, Fujian, China","institution_ids":["https://openalex.org/I2800556661"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032221039","display_name":"Junwen Wu","orcid":null},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiarui Wu","raw_affiliation_strings":["Institute of Artificial Intelligence, Xiamen University, Xiamen, China","Institute of Artificial Intelligence, Xiamen University, China"],"affiliations":[{"raw_affiliation_string":"Institute of Artificial Intelligence, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Institute of Artificial Intelligence, Xiamen University, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079447591","display_name":"Xiaoying Han","orcid":"https://orcid.org/0000-0002-4586-3534"},"institutions":[{"id":"https://openalex.org/I2800556661","display_name":"China Tobacco","ror":"https://ror.org/030d08e08","country_code":"CN","type":"government","lineage":["https://openalex.org/I2800556661"]},{"id":"https://openalex.org/I4210118162","display_name":"Fujian Tobacco Industry Limited Liability Company (China)","ror":"https://ror.org/02fa7s823","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210118162"]},{"id":"https://openalex.org/I4210166335","display_name":"Xiamen Tobacco Industry (China)","ror":"https://ror.org/05t1nkw30","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210166335"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoying Han","raw_affiliation_strings":["Information Center, China Tobacco Fujian Industrial Company Ltd., Xiamen, Fujian, China","Information Center, China Tobacco Fujian Industrial Co., Ltd, Xiamen, Fujian, China"],"affiliations":[{"raw_affiliation_string":"Information Center, China Tobacco Fujian Industrial Company Ltd., Xiamen, Fujian, China","institution_ids":["https://openalex.org/I4210118162","https://openalex.org/I4210166335"]},{"raw_affiliation_string":"Information Center, China Tobacco Fujian Industrial Co., Ltd, Xiamen, Fujian, China","institution_ids":["https://openalex.org/I2800556661"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101883368","display_name":"Qian Gu","orcid":"https://orcid.org/0000-0002-7394-5238"},"institutions":[{"id":"https://openalex.org/I2800556661","display_name":"China Tobacco","ror":"https://ror.org/030d08e08","country_code":"CN","type":"government","lineage":["https://openalex.org/I2800556661"]},{"id":"https://openalex.org/I4210118162","display_name":"Fujian Tobacco Industry Limited Liability Company (China)","ror":"https://ror.org/02fa7s823","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210118162"]},{"id":"https://openalex.org/I4210166335","display_name":"Xiamen Tobacco Industry (China)","ror":"https://ror.org/05t1nkw30","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210166335"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qian Gu","raw_affiliation_strings":["Information Center, China Tobacco Fujian Industrial Company Ltd., Xiamen, Fujian, China","Information Center, China Tobacco Fujian Industrial Co., Ltd, Xiamen, Fujian, China"],"affiliations":[{"raw_affiliation_string":"Information Center, China Tobacco Fujian Industrial Company Ltd., Xiamen, Fujian, China","institution_ids":["https://openalex.org/I4210118162","https://openalex.org/I4210166335"]},{"raw_affiliation_string":"Information Center, China Tobacco Fujian Industrial Co., Ltd, Xiamen, Fujian, China","institution_ids":["https://openalex.org/I2800556661"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000389309","display_name":"Fei Chao","orcid":"https://orcid.org/0000-0002-6928-2638"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fei Chao","raw_affiliation_strings":["Institute of Artificial Intelligence, Xiamen University, Xiamen, China","Institute of Artificial Intelligence, Xiamen University, China"],"affiliations":[{"raw_affiliation_string":"Institute of Artificial Intelligence, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Institute of Artificial Intelligence, Xiamen University, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100399515","display_name":"Xiao Yang","orcid":"https://orcid.org/0000-0002-6690-8439"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiao Yang","raw_affiliation_strings":["School of Informatics, Xiamen University, Xiamen, China","School of Informatics, Xiamen University, China"],"affiliations":[{"raw_affiliation_string":"School of Informatics, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"School of Informatics, Xiamen University, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082402770","display_name":"Fan Wu","orcid":"https://orcid.org/0000-0003-3615-1217"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fan Wu","raw_affiliation_strings":["School of Informatics, Xiamen University, Xiamen, China","School of Informatics, Xiamen University, China"],"affiliations":[{"raw_affiliation_string":"School of Informatics, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"School of Informatics, Xiamen University, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5114949071","display_name":"X Fu","orcid":"https://orcid.org/0000-0001-7958-8684"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Fu","raw_affiliation_strings":["Management School, Xiamen University, Xiamen, China","Xiamen University, China"],"affiliations":[{"raw_affiliation_string":"Management School, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Xiamen University, China","institution_ids":["https://openalex.org/I191208505"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5101201597"],"corresponding_institution_ids":["https://openalex.org/I2800556661","https://openalex.org/I4210118162","https://openalex.org/I4210166335"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.17912764,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"12","issue":null,"first_page":"170541","last_page":"170552"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11338","display_name":"Advancements in Photolithography Techniques","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11338","display_name":"Advancements in Photolithography Techniques","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11301","display_name":"Advanced Surface Polishing Techniques","score":0.984000027179718,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11451","display_name":"Advanced Machining and Optimization Techniques","score":0.9825999736785889,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5963405966758728}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5963405966758728}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2024.3498904","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2024.3498904","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:99ae6ac2d9c441e19b2e27f1092ddf6d","is_oa":true,"landing_page_url":"https://doaj.org/article/99ae6ac2d9c441e19b2e27f1092ddf6d","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 12, Pp 170541-170552 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2024.3498904","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2024.3498904","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7139331918","display_name":null,"funder_award_id":"22YJA630020","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G885185609","display_name":null,"funder_award_id":"72471203","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":37,"referenced_works":["https://openalex.org/W2156150815","https://openalex.org/W2466175319","https://openalex.org/W2606964149","https://openalex.org/W2981751377","https://openalex.org/W2996908057","https://openalex.org/W3168894390","https://openalex.org/W4285179967","https://openalex.org/W4285294723","https://openalex.org/W4312934081","https://openalex.org/W4313341853","https://openalex.org/W4324125328","https://openalex.org/W4360596917","https://openalex.org/W4382998692","https://openalex.org/W4387986939","https://openalex.org/W4388555945","https://openalex.org/W4390818801","https://openalex.org/W6727099177","https://openalex.org/W6749838110","https://openalex.org/W6772383348","https://openalex.org/W6778883912","https://openalex.org/W6789240164","https://openalex.org/W6796581206","https://openalex.org/W6803815625","https://openalex.org/W6810738896","https://openalex.org/W6811340617","https://openalex.org/W6842795023","https://openalex.org/W6846002521","https://openalex.org/W6846164622","https://openalex.org/W6848451824","https://openalex.org/W6850625674","https://openalex.org/W6852962002","https://openalex.org/W6854094408","https://openalex.org/W6854866820","https://openalex.org/W6857663972","https://openalex.org/W6858266299","https://openalex.org/W6859674693","https://openalex.org/W6868668724"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"To":[0],"address":[1],"the":[2,5,11,49,75,85,91,109,113,119,127,134,143,155,175,179],"issue":[3],"of":[4,14,23,52,64,129,178],"substantial":[6],"computational":[7,162],"resource":[8],"consumption":[9],"during":[10],"inference":[12,176],"phase":[13],"large":[15,36,53],"language":[16,54],"models":[17,37,55],"due":[18],"to":[19],"their":[20],"vast":[21],"number":[22],"parameters,":[24],"model":[25,86,158],"sparsification":[26,33,51,145],"is":[27],"an":[28],"effective":[29],"solution.":[30],"However,":[31],"current":[32],"methods":[34],"for":[35,48],"are":[38],"costly.":[39],"We":[40],"propose":[41],"a":[42,57,96,165],"comprehensive":[43],"two-stage":[44],"approach":[45,62],"called":[46],"ELO-Mask":[47],"rapid":[50],"using":[56,77,90,118,164],"small":[58,97,121],"calibration":[59],"dataset.":[60,99,123],"The":[61],"consists":[63],"two":[65],"steps:":[66],"1)":[67],"Mask":[68,101],"Reordering":[69],"Step,":[70,103],"this":[71,104,130],"step":[72,105,115],"involves":[73,106],"initializing":[74],"mask":[76],"predefined":[78],"parameter":[79],"importance":[80],"metrics,":[81],"followed":[82],"by":[83],"reordering":[84],"masks":[87,110],"in":[88,116,154],"blocks":[89],"Straight-Through":[92],"Estimator":[93],"method":[94,138],"with":[95],"sample":[98,122],"2)":[100],"Fine-Tuning":[102],"further":[107],"fine-tuning":[108,148],"obtained":[111],"from":[112],"first":[114],"blocks,":[117],"same":[120],"Our":[124],"experiments":[125],"demonstrate":[126],"effectiveness":[128],"approach.":[131,149],"When":[132],"sparsifying":[133],"Llama-7B":[135],"model,":[136],"our":[137],"shows":[139],"significant":[140],"superiority":[141],"over":[142],"standard":[144],"plus":[146],"LoRA":[147],"It":[150],"achieves":[151],"comparable":[152],"performance":[153],"final":[156],"sparse":[157,180],"while":[159],"consuming":[160],"less":[161,169],"power,":[163],"smaller":[166],"dataset,":[167],"occupying":[168],"GPU":[170],"memory,":[171],"and":[172],"not":[173],"affecting":[174],"speed":[177],"model.":[181]},"counts_by_year":[],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-10T00:00:00"}