{"id":"https://openalex.org/W4412888157","doi":"https://doi.org/10.18653/v1/2025.findings-acl.917","title":"C2KD: Cross-layer and Cross-head Knowledge Distillation for Small Language Model-based Recommendation","display_name":"C2KD: Cross-layer and Cross-head Knowledge Distillation for Small Language Model-based Recommendation","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412888157","doi":"https://doi.org/10.18653/v1/2025.findings-acl.917"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.917","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.917","pdf_url":"https://aclanthology.org/2025.findings-acl.917.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.917.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100373704","display_name":"Xiao Chen","orcid":"https://orcid.org/0000-0002-1667-4531"},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiao Chen","raw_affiliation_strings":["The Hong Kong Polytechnic University  Center for Artificial Intelligence and Robotics , HKISI-CAS  Institute of Automation , CAS  University of Chinese Academy of Sciences"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University  Center for Artificial Intelligence and Robotics , HKISI-CAS  Institute of Automation , CAS  University of Chinese Academy of Sciences","institution_ids":["https://openalex.org/I4210112150"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036449631","display_name":"Changyi Ma","orcid":"https://orcid.org/0000-0002-4617-5821"},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Changyi Ma","raw_affiliation_strings":["The Hong Kong Polytechnic University  Center for Artificial Intelligence and Robotics , HKISI-CAS  Institute of Automation , CAS  University of Chinese Academy of Sciences"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University  Center for Artificial Intelligence and Robotics , HKISI-CAS  Institute of Automation , CAS  University of Chinese Academy of Sciences","institution_ids":["https://openalex.org/I4210112150"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100583858","display_name":"Wenqi Fan","orcid":"https://orcid.org/0009-0003-9396-0835"},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenqi Fan","raw_affiliation_strings":["The Hong Kong Polytechnic University  Center for Artificial Intelligence and Robotics , HKISI-CAS  Institute of Automation , CAS  University of Chinese Academy of Sciences"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University  Center for Artificial Intelligence and Robotics , HKISI-CAS  Institute of Automation , CAS  University of Chinese Academy of Sciences","institution_ids":["https://openalex.org/I4210112150"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101589395","display_name":"Zhaoxiang Zhang","orcid":"https://orcid.org/0009-0002-5344-7159"},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhaoxiang Zhang","raw_affiliation_strings":["The Hong Kong Polytechnic University  Center for Artificial Intelligence and Robotics , HKISI-CAS  Institute of Automation , CAS  University of Chinese Academy of Sciences"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University  Center for Artificial Intelligence and Robotics , HKISI-CAS  Institute of Automation , CAS  University of Chinese Academy of Sciences","institution_ids":["https://openalex.org/I4210112150"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100404176","display_name":"Qing Li","orcid":"https://orcid.org/0000-0003-3370-471X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li Qing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"17827","last_page":"17838"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.982200026512146,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9799000024795532,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7063631415367126},{"id":"https://openalex.org/keywords/head","display_name":"Head (geology)","score":0.6147764325141907},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.5429217219352722},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.528044581413269},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3633986711502075},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.36151862144470215},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.09408211708068848},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.07737073302268982},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.05756664276123047},{"id":"https://openalex.org/keywords/chromatography","display_name":"Chromatography","score":0.05679824948310852},{"id":"https://openalex.org/keywords/composite-material","display_name":"Composite material","score":0.05572468042373657}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7063631415367126},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.6147764325141907},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.5429217219352722},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.528044581413269},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3633986711502075},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.36151862144470215},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.09408211708068848},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.07737073302268982},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.05756664276123047},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.05679824948310852},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.05572468042373657},{"id":"https://openalex.org/C114793014","wikidata":"https://www.wikidata.org/wiki/Q52109","display_name":"Geomorphology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.917","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.917","pdf_url":"https://aclanthology.org/2025.findings-acl.917.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.917","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.917","pdf_url":"https://aclanthology.org/2025.findings-acl.917.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.4300000071525574,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G4078110728","display_name":null,"funder_award_id":"P0048625","funder_id":"https://openalex.org/F4320322598","funder_display_name":"Hong Kong Polytechnic University"},{"id":"https://openalex.org/G528947399","display_name":null,"funder_award_id":"P0042693","funder_id":"https://openalex.org/F4320322598","funder_display_name":"Hong Kong Polytechnic University"}],"funders":[{"id":"https://openalex.org/F4320322598","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412888157.pdf","grobid_xml":"https://content.openalex.org/works/W4412888157.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3097502728","https://openalex.org/W3026162553","https://openalex.org/W2113206756","https://openalex.org/W2344382886","https://openalex.org/W19111321","https://openalex.org/W2412887479","https://openalex.org/W2953684491","https://openalex.org/W4285338581","https://openalex.org/W2768175398","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Sequential":[0],"recommenders":[1],"predict":[2],"users'":[3],"next":[4],"interactions":[5],"based":[6],"on":[7,31,160],"historical":[8],"behavior":[9],"and":[10,25,53,89,146,165,190],"are":[11],"essential":[12],"in":[13],"modern":[14],"recommendation":[15,48,91],"systems.While":[16],"Large":[17],"Language":[18,34],"Models":[19,35],"(LLMs)":[20],"show":[21],"promise,":[22],"their":[23],"size":[24],"high":[26],"inference":[27],"costs":[28],"limit":[29],"deployment":[30],"resource-constrained":[32],"devices.Small":[33],"(SLMs)":[36],"provide":[37],"a":[38,113,136,215],"more":[39],"efficient":[40],"alternative":[41],"for":[42,218],"edge":[43],"devices,":[44],"but":[45],"bridging":[46],"the":[47,141,151,156,161,172,176,179,188,191],"performance":[49,70,208],"gap":[50],"between":[51,187],"LLMs":[52,211],"SLMs":[54,204],"remains":[55],"challenging.Typical":[56],"approaches":[57],"like":[58],"supervised":[59],"fine-tuning":[60],"or":[61,71],"vanilla":[62,81],"knowledge":[63,122,149],"distillation":[64],"(KD)":[65],"often":[66],"lead":[67],"to":[68,119,139,158,178,205,210],"suboptimal":[69],"even":[72],"negative":[73],"transfer.Our":[74],"motivational":[75],"experiments":[76,193],"reveal":[77],"key":[78],"issues":[79],"with":[80],"KD":[82],"methods:":[83],"feature":[84,132],"imitation":[85],"suffers":[86],"from":[87,123,150],"redundancy":[88],"uneven":[90],"ability":[92],"across":[93,194],"layers,":[94,154],"while":[95],"prediction":[96,106,185],"mimicking":[97],"faces":[98],"conflicts":[99],"caused":[100],"by":[101],"differing":[102],"weight":[103],"distributions":[104],"of":[105,175],"heads.To":[107],"address":[108],"these":[109],"challenges,":[110],"we":[111],"propose":[112],"simple":[114],"yet":[115],"effective":[116],"framework,":[117],"C2KD,":[118],"transfer":[120],"taskrelevant":[121],"two":[124],"complementary":[125],"dimensions.Specifically,":[126],"our":[127,200],"method":[128],"incorporates:":[129],"(1)":[130],"cross-layer":[131],"imitation,":[133],"which":[134,170],"uses":[135],"dynamic":[137],"router":[138],"select":[140],"most":[142],"relevant":[143],"teacher":[144,189],"layers":[145],"assimilate":[147],"task-relevant":[148],"teacher's":[152,162,180],"late":[153],"allowing":[155],"student":[157,177],"concentrate":[159],"specialized":[163],"knowledge;":[164],"(2)":[166],"cross-head":[167],"logit":[168],"distillation,":[169],"maps":[171],"intermediate":[173],"features":[174],"output":[181],"head,":[182],"thereby":[183],"minimizing":[184],"discrepancies":[186],"student.Extensive":[192],"diverse":[195],"model":[196],"families":[197],"demonstrate":[198],"that":[199],"approach":[201],"enables":[202],"1Bparameter":[203],"achieve":[206],"competitive":[207],"compared":[209],"(e.g.,":[212],"Llama3-8B),":[213],"offering":[214],"practical":[216],"solution":[217],"real-world":[219],"ondevice":[220],"sequential":[221],"recommendations.":[222]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}