{"id":"https://openalex.org/W4403747428","doi":"https://doi.org/10.48550/arxiv.2409.12545","title":"Enhancing Knowledge Distillation of Large Language Models through Efficient Multi-Modal Distribution Alignment","display_name":"Enhancing Knowledge Distillation of Large Language Models through Efficient Multi-Modal Distribution Alignment","publication_year":2024,"publication_date":"2024-09-19","ids":{"openalex":"https://openalex.org/W4403747428","doi":"https://doi.org/10.48550/arxiv.2409.12545"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2409.12545","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.12545","pdf_url":"https://arxiv.org/pdf/2409.12545","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2409.12545","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Peng, Tianyu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Tianyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5102786071","display_name":"Jiajun Zhang","orcid":"https://orcid.org/0009-0008-1989-1196","observed_orcids":["https://orcid.org/0009-0008-1989-1196"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Jiajun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.7807999849319458,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.7807999849319458,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.14429999887943268,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.05869999900460243,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":1.0},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9789999723434448},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.7379999756813049},{"id":"https://openalex.org/keywords/ranking-loss","display_name":"ranking loss","score":0.8939999938011169},{"id":"https://openalex.org/keywords/teacher-student-models","display_name":"teacher-student models","score":0.6060000061988831}],"concepts":[{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.7405391335487366},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5852516293525696},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5841379761695862},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.5324738025665283},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3808900713920593},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.336176335811615},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1965162456035614},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.11302277445793152},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.07641071081161499},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2409.12545","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.12545","pdf_url":"https://arxiv.org/pdf/2409.12545","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2409.12545","doi":"https://doi.org/10.48550/arxiv.2409.12545","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2409.12545","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2409.12545","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.12545","pdf_url":"https://arxiv.org/pdf/2409.12545","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.5}],"study_designs":[],"awards":[{"id":"https://openalex.org/G4874090527","display_name":null,"funder_award_id":"62122088","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403747428.pdf","grobid_xml":"https://content.openalex.org/works/W4403747428.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4407719547","https://openalex.org/W4388717304","https://openalex.org/W7221552649","https://openalex.org/W4416052937","https://openalex.org/W4402808682","https://openalex.org/W7150904884","https://openalex.org/W4417081414","https://openalex.org/W7168171913","https://openalex.org/W7131879468","https://openalex.org/W3208239235"],"abstract_inverted_index":{"Knowledge":[0,75],"distillation":[1,105],"(KD)":[2],"is":[3],"an":[4],"effective":[5],"model":[6,131],"compression":[7],"method":[8,127],"that":[9,125],"can":[10],"transfer":[11],"the":[12,24,45,56,80,83,89,110,129,135,139],"internal":[13],"capabilities":[14],"of":[15,47,58,82,85,118,138],"large":[16],"language":[17],"models":[18,36],"(LLMs)":[19],"to":[20,37,132,143],"smaller":[21],"ones.":[22],"However,":[23],"multi-modal":[25,48,64,136],"probability":[26],"distribution":[27,49],"predicted":[28,120],"by":[29],"teacher":[30,90,140],"LLMs":[31],"causes":[32],"difficulties":[33],"for":[34],"student":[35,92,130],"learn.":[38],"In":[39],"this":[40,68],"paper,":[41],"we":[42,70,99],"first":[43],"demonstrate":[44,124],"importance":[46],"alignment":[50],"with":[51,103],"experiments":[52],"and":[53,91],"then":[54],"highlight":[55],"inefficiency":[57],"existing":[59,104],"KD":[60],"approaches":[61],"in":[62,116,148],"learning":[63],"distributions.":[65],"To":[66],"address":[67],"problem,":[69],"propose":[71],"Ranking":[72],"Loss":[73],"based":[74],"Distillation":[76],"(RLKD),":[77],"which":[78],"encourages":[79],"consistency":[81],"ranking":[84,97],"peak":[86],"predictions":[87],"between":[88,113],"models.":[93],"By":[94],"incorporating":[95],"word-level":[96],"loss,":[98],"ensure":[100],"excellent":[101],"compatibility":[102],"objectives":[106],"while":[107],"fully":[108],"leveraging":[109],"fine-grained":[111],"information":[112],"different":[114],"categories":[115],"peaks":[117],"two":[119],"distribution.":[121],"Experimental":[122],"results":[123],"our":[126],"enables":[128],"better":[133],"learn":[134],"distributions":[137],"model,":[141],"leading":[142],"a":[144],"significant":[145],"performance":[146],"improvement":[147],"various":[149],"downstream":[150],"tasks.":[151]},"counts_by_year":[],"updated_date":"2026-09-29T14:09:16.559479","created_date":"2024-10-25T00:00:00"}