{"id":"https://openalex.org/W4407953206","doi":"https://doi.org/10.1145/3701551.3703577","title":"Beyond Answers: Transferring Reasoning Capabilities to Smaller LLMs Using Multi-Teacher Knowledge Distillation","display_name":"Beyond Answers: Transferring Reasoning Capabilities to Smaller LLMs Using Multi-Teacher Knowledge Distillation","publication_year":2025,"publication_date":"2025-02-26","ids":{"openalex":"https://openalex.org/W4407953206","doi":"https://doi.org/10.1145/3701551.3703577"},"language":"en","primary_location":{"id":"doi:10.1145/3701551.3703577","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3701551.3703577","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Eighteenth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3701551.3703577","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057838053","display_name":"Yijun Tian","orcid":"https://orcid.org/0000-0003-2795-6080"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yijun Tian","raw_affiliation_strings":["University of Notre Dame, Notre Dame, USA"],"raw_orcid":"https://orcid.org/0000-0003-2795-6080","affiliations":[{"raw_affiliation_string":"University of Notre Dame, Notre Dame, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037571863","display_name":"Yikun Han","orcid":"https://orcid.org/0009-0002-9683-3092"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yikun Han","raw_affiliation_strings":["University of Michigan, Ann Arbor, USA"],"raw_orcid":"https://orcid.org/0009-0002-9683-3092","affiliations":[{"raw_affiliation_string":"University of Michigan, Ann Arbor, USA","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049591203","display_name":"Xiusi Chen","orcid":"https://orcid.org/0000-0002-9713-8000"},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiusi Chen","raw_affiliation_strings":["University of California, Los Angeles, Los Angeles, USA"],"raw_orcid":"https://orcid.org/0000-0002-9713-8000","affiliations":[{"raw_affiliation_string":"University of California, Los Angeles, Los Angeles, USA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100392089","display_name":"Wei Wang","orcid":"https://orcid.org/0000-0002-8180-2886"},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wei Wang","raw_affiliation_strings":["University of California, Los Angeles, Los Angeles, USA"],"raw_orcid":"https://orcid.org/0000-0002-8180-2886","affiliations":[{"raw_affiliation_string":"University of California, Los Angeles, Los Angeles, USA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068157871","display_name":"Nitesh V. Chawla","orcid":"https://orcid.org/0000-0003-3932-5956"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nitesh V. Chawla","raw_affiliation_strings":["University of Notre Dame, Notre Dame, USA"],"raw_orcid":"https://orcid.org/0000-0003-3932-5956","affiliations":[{"raw_affiliation_string":"University of Notre Dame, Notre Dame, USA","institution_ids":["https://openalex.org/I107639228"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":27.6769,"has_fulltext":false,"cited_by_count":16,"citation_normalized_percentile":{"value":0.99713769,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"251","last_page":"260"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9503999948501587,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9503999948501587,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9437999725341797,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12171","display_name":"Open Education and E-Learning","score":0.9153000116348267,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5596538782119751},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5460463166236877},{"id":"https://openalex.org/keywords/case-based-reasoning","display_name":"Case-based reasoning","score":0.4139772653579712},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.3283870220184326},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2850020229816437},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1257413923740387},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.11421915888786316},{"id":"https://openalex.org/keywords/chromatography","display_name":"Chromatography","score":0.06754675507545471}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5596538782119751},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5460463166236877},{"id":"https://openalex.org/C20162079","wikidata":"https://www.wikidata.org/wiki/Q1151406","display_name":"Case-based reasoning","level":2,"score":0.4139772653579712},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3283870220184326},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2850020229816437},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1257413923740387},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.11421915888786316},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.06754675507545471}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3701551.3703577","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3701551.3703577","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Eighteenth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3701551.3703577","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3701551.3703577","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Eighteenth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.5799999833106995}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W1981208470","https://openalex.org/W2890894339","https://openalex.org/W2891012317","https://openalex.org/W2962767298","https://openalex.org/W2963082289","https://openalex.org/W2970482702","https://openalex.org/W2990138404","https://openalex.org/W3034368386","https://openalex.org/W3126739176","https://openalex.org/W3138154797","https://openalex.org/W3174770825","https://openalex.org/W3212191244","https://openalex.org/W4205737716","https://openalex.org/W4205991051","https://openalex.org/W4288055447","https://openalex.org/W4385571011","https://openalex.org/W4385571944","https://openalex.org/W4389519494","https://openalex.org/W4389519995","https://openalex.org/W4389520139","https://openalex.org/W4389523624","https://openalex.org/W4389523706","https://openalex.org/W4392367398","https://openalex.org/W4392669753","https://openalex.org/W4393147025","https://openalex.org/W4402667018","https://openalex.org/W6600031136","https://openalex.org/W6600351811","https://openalex.org/W6601865935","https://openalex.org/W6816482620"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Transferring":[0],"the":[1,29,55,63,67,96,103,109,124,149,169],"reasoning":[2,120,166],"capability":[3],"from":[4,47,87,130],"stronger":[5],"large":[6,89,180],"language":[7,71],"models":[8],"(LLMs)":[9],"to":[10,23,37,81,99,127,146],"smaller":[11,18,187],"ones":[12],"has":[13],"been":[14],"quite":[15],"appealing,":[16],"as":[17],"LLMs":[19,117,182],"are":[20,151],"more":[21],"flexible":[22],"deploy":[24],"with":[25],"less":[26],"expense.":[27],"Among":[28],"existing":[30,44],"solutions,":[31],"knowledge":[32,52,78,129],"distillation":[33,79],"stands":[34],"out":[35],"due":[36],"its":[38],"outstanding":[39],"efficiency":[40],"and":[41,54,65,141,153],"generalization.":[42],"However,":[43],"methods":[45],"suffer":[46],"several":[48],"drawbacks,":[49],"including":[50],"limited":[51],"diversity":[53],"lack":[56],"of":[57,69,171],"rich":[58],"contextual":[59],"information.":[60],"To":[61],"solve":[62],"problems":[64],"facilitate":[66],"learning":[68],"compact":[70],"models,":[72],"we":[73,94,122],"propose":[74],"TinyLLM,":[75],"a":[76,83,142,185],"new":[77],"paradigm":[80],"learn":[82],"small":[84],"student":[85,97,125],"LLM":[86,98],"multiple":[88],"teacher":[90,132,181],"LLMs.":[91,133],"In":[92],"particular,":[93],"encourage":[95],"not":[100],"only":[101],"generate":[102],"correct":[104],"answers":[105],"but":[106],"also":[107],"understand":[108],"rationales":[110,150],"behind":[111],"these":[112],"answers.":[113],"Given":[114],"that":[115,148,176],"different":[116],"possess":[118],"diverse":[119],"skills,":[121],"guide":[123],"model":[126,188],"assimilate":[128],"various":[131],"We":[134],"further":[135],"introduce":[136],"an":[137],"in-context":[138],"example":[139],"generator":[140],"teacher-forcing":[143],"Chain-of-Thought":[144],"strategy":[145],"ensure":[147],"accurate":[152],"grounded":[154],"in":[155],"contextually":[156],"appropriate":[157],"scenarios.":[158],"Extensive":[159],"experiments":[160],"on":[161],"six":[162],"datasets":[163],"across":[164],"two":[165],"tasks":[167],"demonstrate":[168],"superiority":[170],"our":[172],"method.":[173],"Results":[174],"show":[175],"TinyLLM":[177],"can":[178],"outperform":[179],"significantly,":[183],"despite":[184],"considerably":[186],"size.":[189],"The":[190],"source":[191],"code":[192],"is":[193],"available":[194],"at:":[195],"https://github.com/YikunHan42/TinyLLM.":[196]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":8}],"updated_date":"2026-08-12T21:12:35.861297","created_date":"2025-10-10T00:00:00"}