{"id":"https://openalex.org/W4385767949","doi":"https://doi.org/10.24963/ijcai.2023/568","title":"Towards Lossless Head Pruning through Automatic Peer Distillation for Language Models","display_name":"Towards Lossless Head Pruning through Automatic Peer Distillation for Language Models","publication_year":2023,"publication_date":"2023-08-01","ids":{"openalex":"https://openalex.org/W4385767949","doi":"https://doi.org/10.24963/ijcai.2023/568"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2023/568","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2023/568","pdf_url":"https://www.ijcai.org/proceedings/2023/0568.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.ijcai.org/proceedings/2023/0568.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100375966","display_name":"Bingbing Li","orcid":"https://orcid.org/0000-0002-9224-2685"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bingbing Li","raw_affiliation_strings":["University of Connecticut"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Connecticut","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009344627","display_name":"Zigeng Wang","orcid":"https://orcid.org/0000-0001-5611-2165"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zigeng Wang","raw_affiliation_strings":["University of Connecticut"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Connecticut","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100529062","display_name":"Shaoyi Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shaoyi Huang","raw_affiliation_strings":["University of Connecticut"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Connecticut","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023766412","display_name":"Mikhail A. Bragin","orcid":"https://orcid.org/0000-0002-7783-9053"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mikhail Bragin","raw_affiliation_strings":["University of Connecticut"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Connecticut","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100421143","display_name":"Ji Li","orcid":"https://orcid.org/0000-0001-7910-8011"},"institutions":[{"id":"https://openalex.org/I4210105678","display_name":"Microsoft (Finland)","ror":"https://ror.org/01nehjf29","country_code":"FI","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210105678"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Ji Li","raw_affiliation_strings":["Microsoft Corporation"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation","institution_ids":["https://openalex.org/I4210105678"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5030060072","display_name":"Caiwen Ding","orcid":"https://orcid.org/0000-0003-0891-1231"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Caiwen Ding","raw_affiliation_strings":["University of Connecticut"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Connecticut","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"5113","last_page":"5121"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8116305470466614},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.714247465133667},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.6014902591705322},{"id":"https://openalex.org/keywords/hyperparameter","display_name":"Hyperparameter","score":0.5685005187988281},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5288451910018921},{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.4735504686832428},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.46248775720596313},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.45922863483428955},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.4494011402130127},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.34409576654434204},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.08526110649108887}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8116305470466614},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.714247465133667},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.6014902591705322},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.5685005187988281},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5288451910018921},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.4735504686832428},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.46248775720596313},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45922863483428955},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.4494011402130127},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34409576654434204},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.08526110649108887},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C6557445","wikidata":"https://www.wikidata.org/wiki/Q173113","display_name":"Agronomy","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.24963/ijcai.2023/568","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2023/568","pdf_url":"https://www.ijcai.org/proceedings/2023/0568.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2023/568","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2023/568","pdf_url":"https://www.ijcai.org/proceedings/2023/0568.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/8","score":0.7300000190734863,"display_name":"Decent work and economic growth"}],"awards":[{"id":"https://openalex.org/G4146176377","display_name":null,"funder_award_id":"2022-67023-36399","funder_id":"https://openalex.org/F4320332299","funder_display_name":"National Institute of Food and Agriculture"}],"funders":[{"id":"https://openalex.org/F4320306114","display_name":"U.S. Department of Agriculture","ror":"https://ror.org/01na82s61"},{"id":"https://openalex.org/F4320332299","display_name":"National Institute of Food and Agriculture","ror":"https://ror.org/05qx3fv49"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385767949.pdf"},"referenced_works_count":22,"referenced_works":["https://openalex.org/W2194775991","https://openalex.org/W2551895583","https://openalex.org/W2896457183","https://openalex.org/W2923014074","https://openalex.org/W2946794439","https://openalex.org/W2962965870","https://openalex.org/W2963082137","https://openalex.org/W2963350559","https://openalex.org/W2970120757","https://openalex.org/W2970597249","https://openalex.org/W2970860468","https://openalex.org/W2980282514","https://openalex.org/W3042215422","https://openalex.org/W3100980998","https://openalex.org/W3106483960","https://openalex.org/W3106504817","https://openalex.org/W3159727696","https://openalex.org/W3166574921","https://openalex.org/W3176609328","https://openalex.org/W4286902761","https://openalex.org/W4292363360","https://openalex.org/W4295262505"],"related_works":["https://openalex.org/W4288365749","https://openalex.org/W2936497627","https://openalex.org/W3013624417","https://openalex.org/W4287826556","https://openalex.org/W3098382480","https://openalex.org/W4287598411","https://openalex.org/W3100913109","https://openalex.org/W3198458223","https://openalex.org/W3126642501","https://openalex.org/W2964413124"],"abstract_inverted_index":{"Pruning":[0],"has":[1],"been":[2],"extensively":[3],"studied":[4],"in":[5,113,121],"Transformer-based":[6],"language":[7,67],"models":[8,68],"to":[9,25,43,89,107],"improve":[10,26],"efficiency.":[11],"Typically,":[12],"we":[13,32,51],"zero":[14],"(prune)":[15],"unimportant":[16],"model":[17,24,45,149],"weights":[18],"and":[19,37,69,82,85,163],"train":[20],"a":[21,61,87],"derived":[22],"compact":[23],"final":[27],"accuracy.":[28],"For":[29],"pruned":[30,79,96,143],"weights,":[31],"treat":[33],"them":[34],"as":[35,57],"useless":[36],"discard":[38],"them.":[39],"This":[40],"usually":[41],"leads":[42],"significant":[44],"accuracy":[46],"degradation.":[47],"In":[48],"this":[49],"paper,":[50],"focus":[52],"on":[53,125,161],"attention":[54,59,80,111],"head":[55,58],"pruning":[56],"is":[60],"key":[62],"component":[63],"of":[64],"the":[65,76,91,95,109,117,126,145],"transformer-based":[66],"provides":[70],"interpretable":[71],"knowledge":[72,93,141],"meaning.":[73],"We":[74,101],"reveal":[75],"relationship":[77],"between":[78],"heads":[81,84,112,157],"retained":[83],"provide":[86],"solution":[88],"recycle":[90],"discarded":[92,140],"from":[94,142],"heads,":[97,144],"named":[98],"peer":[99],"distillation.":[100],"also":[102],"develop":[103],"an":[104],"automatic":[105],"framework":[106],"locate":[108],"to-be-pruned":[110],"each":[114],"layer,":[115],"freeing":[116],"time-consuming":[118],"human":[119],"labor":[120],"tuning":[122],"hyperparameters.Experimental":[123],"results":[124],"General":[127],"Language":[128],"Understanding":[129],"Evaluation":[130],"(GLUE)":[131],"benchmark":[132],"are":[133],"provided":[134],"using":[135],"BERT":[136],"model.":[137],"By":[138],"recycling":[139],"proposed":[146],"method":[147],"maintains":[148],"performance":[150],"across":[151],"all":[152],"nine":[153],"tasks":[154],"while":[155],"reducing":[156],"by":[158],"over":[159],"58%":[160],"average":[162],"outperforms":[164],"state-of-the-art":[165],"techniques":[166],"(e.g.,":[167],"Random,":[168],"HISP,":[169],"L0":[170],"Norm,":[171],"SMP).":[172]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}