{"id":"https://openalex.org/W7134817535","doi":"https://doi.org/10.1109/tkde.2026.3671872","title":"Honey, I Shrunk the Language Model: Impact of Knowledge Distillation Methods on Performance and Explainability","display_name":"Honey, I Shrunk the Language Model: Impact of Knowledge Distillation Methods on Performance and Explainability","publication_year":2026,"publication_date":"2026-03-09","ids":{"openalex":"https://openalex.org/W7134817535","doi":"https://doi.org/10.1109/tkde.2026.3671872"},"language":null,"primary_location":{"id":"doi:10.1109/tkde.2026.3671872","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2026.3671872","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Daniel Hendriks","orcid":"https://orcid.org/0009-0002-3294-827X"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Daniel Hendriks","raw_affiliation_strings":["Institute for Information Systems (WIN), Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0009-0002-3294-827X","affiliations":[{"raw_affiliation_string":"Institute for Information Systems (WIN), Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128665307","display_name":"Philipp Spitzer","orcid":null},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Philipp Spitzer","raw_affiliation_strings":["Institute for Information Systems (WIN), Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0000-0002-9378-0872","affiliations":[{"raw_affiliation_string":"Institute for Information Systems (WIN), Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128655828","display_name":"Niklas K\u00fchl","orcid":null},"institutions":[{"id":"https://openalex.org/I54009628","display_name":"University of Bayreuth","ror":"https://ror.org/0234wmv40","country_code":"DE","type":"education","lineage":["https://openalex.org/I54009628"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Niklas K\u00fchl","raw_affiliation_strings":["Information Systems (WI) Institute, University of Bayreuth, Bayreuth, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Information Systems (WI) Institute, University of Bayreuth, Bayreuth, Germany","institution_ids":["https://openalex.org/I54009628"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5000245557","display_name":"Gerhard Satzger","orcid":"https://orcid.org/0000-0001-8731-654X"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Gerhard Satzger","raw_affiliation_strings":["Institute for Information Systems (WIN), Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0000-0001-8731-654X","affiliations":[{"raw_affiliation_string":"Institute for Information Systems (WIN), Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.31180273,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"38","issue":"6","first_page":"3710","last_page":"3721"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.2531999945640564,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.2531999945640564,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1761000007390976,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.17499999701976776,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.8611999750137329},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.48089998960494995},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.47110000252723694},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.41850000619888306},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.38600000739097595},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.3490000069141388},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"Natural language understanding","score":0.34360000491142273}],"concepts":[{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.8611999750137329},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7980999946594238},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5702999830245972},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5371000170707703},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.48089998960494995},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.47110000252723694},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.41850000619888306},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.38600000739097595},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3490000069141388},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.34360000491142273},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.33329999446868896},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.3330000042915344},{"id":"https://openalex.org/C66833556","wikidata":"https://www.wikidata.org/wiki/Q1863368","display_name":"Continuous distillation","level":4,"score":0.3296000063419342},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.31630000472068787},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.31299999356269836},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.28760001063346863}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tkde.2026.3671872","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2026.3671872","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":66,"referenced_works":["https://openalex.org/W401943522","https://openalex.org/W2061141931","https://openalex.org/W2332488709","https://openalex.org/W2885138528","https://openalex.org/W2927351257","https://openalex.org/W2962833140","https://openalex.org/W2963609017","https://openalex.org/W2964303497","https://openalex.org/W2979826702","https://openalex.org/W2981731882","https://openalex.org/W3000965188","https://openalex.org/W3004127093","https://openalex.org/W3101981467","https://openalex.org/W3102035862","https://openalex.org/W3105966348","https://openalex.org/W3118741274","https://openalex.org/W3133543405","https://openalex.org/W3159959439","https://openalex.org/W3167602185","https://openalex.org/W3174253819","https://openalex.org/W3174510164","https://openalex.org/W3175866696","https://openalex.org/W3175910413","https://openalex.org/W3176693010","https://openalex.org/W3194782062","https://openalex.org/W3207493267","https://openalex.org/W3212191244","https://openalex.org/W4205991051","https://openalex.org/W4251988601","https://openalex.org/W4285225959","https://openalex.org/W4319321305","https://openalex.org/W4320082194","https://openalex.org/W4385059308","https://openalex.org/W4385570561","https://openalex.org/W4385571011","https://openalex.org/W4385571219","https://openalex.org/W4385571260","https://openalex.org/W4385572016","https://openalex.org/W4385572492","https://openalex.org/W4385572836","https://openalex.org/W4385572896","https://openalex.org/W4386032032","https://openalex.org/W4389523945","https://openalex.org/W4389524372","https://openalex.org/W4390871831","https://openalex.org/W4392939508","https://openalex.org/W4396530167","https://openalex.org/W4400772171","https://openalex.org/W4401042410","https://openalex.org/W4401326097","https://openalex.org/W4402654766","https://openalex.org/W4402670312","https://openalex.org/W4402670989","https://openalex.org/W4402683840","https://openalex.org/W4403864719","https://openalex.org/W4404295514","https://openalex.org/W4404314567","https://openalex.org/W4404695057","https://openalex.org/W4404782194","https://openalex.org/W4409503125","https://openalex.org/W4410061029","https://openalex.org/W4412603555","https://openalex.org/W4412886747","https://openalex.org/W7131797834","https://openalex.org/W7131876944","https://openalex.org/W7133209886"],"related_works":[],"abstract_inverted_index":{"Artificial":[0],"Intelligence":[1],"(AI)":[2],"has":[3,52],"increasingly":[4],"influenced":[5],"modern":[6],"society,":[7],"recently":[8],"in":[9,14,31,159],"particular":[10],"through":[11],"significant":[12],"advancements":[13],"Large":[15],"Language":[16,129],"Models":[17],"(LLMs).":[18],"However,":[19],"high":[20],"computational":[21],"and":[22,62,79,86,107,132,156,164,182],"storage":[23],"demands":[24],"of":[25,72,95,161,172,185],"LLMs":[26],"still":[27],"limit":[28],"their":[29,68,157],"deployment":[30],"resource-constrained":[32],"environments.":[33],"Knowledge":[34],"distillation":[35,55,74,103,117,154,171],"addresses":[36],"this":[37,89],"challenge":[38],"by":[39,98,108],"training":[40,60,63,111],"a":[41,46,145],"small":[42,173],"student":[43,65,140],"model":[44,77,141],"from":[45],"larger":[47],"teacher":[48],"model.":[49,66],"Previous":[50],"research":[51],"introduced":[53],"several":[54],"methods":[56,75,97,118,155],"for":[57,104],"both":[58,162],"generating":[59],"data":[61,105],"the":[64,70,93,116,120,170],"Despite":[67],"relevance,":[69],"effects":[71],"state-of-the-art":[73],"on":[76,119],"performance":[78,138,163],"explainability":[80],"have":[81],"not":[82],"been":[83],"thoroughly":[84],"investigated":[85],"compared.":[87],"In":[88],"work,":[90],"we":[91,136,143],"enlarge":[92],"set":[94],"available":[96],"applying":[99],"critique-revision":[100],"prompting":[101],"to":[102,148,179],"generation":[106],"synthesizing":[109],"existing":[110],"methods.":[112],"We":[113,151],"systematically":[114],"compare":[115],"widely":[121],"used":[122],"Commonsense":[123],"Question-Answering":[124],"(CQA),":[125],"Extended":[126],"Stanford":[127],"Natural":[128],"Inference":[130],"(ESNLI),":[131],"StrategyQA":[133],"datasets.":[134],"While":[135],"measure":[137],"via":[139],"accuracy,":[142],"employ":[144],"human-grounded":[146],"study":[147],"evaluate":[149],"explainability.":[150,165],"contribute":[152,178],"new":[153],"comparison":[158],"terms":[160],"This":[166],"should":[167],"further":[168],"advance":[169],"language":[174,186],"models":[175],"and,":[176],"thus,":[177],"broader":[180],"applicability":[181],"faster":[183],"diffusion":[184],"models.":[187]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-03-11T00:00:00"}