{"id":"https://openalex.org/W4414899016","doi":"https://doi.org/10.48550/arxiv.2506.01954","title":"DRAG: Distilling RAG for SLMs from LLMs to Transfer Knowledge and Mitigate Hallucination via Evidence and Graph-based Distillation","display_name":"DRAG: Distilling RAG for SLMs from LLMs to Transfer Knowledge and Mitigate Hallucination via Evidence and Graph-based Distillation","publication_year":2025,"publication_date":"2025-06-02","ids":{"openalex":"https://openalex.org/W4414899016","doi":"https://doi.org/10.48550/arxiv.2506.01954"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2506.01954","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.01954","pdf_url":"https://arxiv.org/pdf/2506.01954","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2506.01954","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042429687","display_name":"J. Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Chen, Jennifer","raw_affiliation_strings":["McGill University \u266e"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"McGill University \u266e","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093598412","display_name":"Aidar Myrzakhan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Myrzakhan, Aidar","raw_affiliation_strings":["University of AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of AI","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Luo, Yaxin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I12003341","display_name":"National University of Science and Technology","ror":"https://ror.org/019vsm959","country_code":"RU","type":"education","lineage":["https://openalex.org/I12003341"]},{"id":"https://openalex.org/I312958679","display_name":"National University of Science and Technology","ror":"https://ror.org/02kesvt12","country_code":"ZW","type":"education","lineage":["https://openalex.org/I312958679"]}],"countries":["RU","ZW"],"is_corresponding":false,"raw_author_name":"Luo, Yaxin","raw_affiliation_strings":["National University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Science and Technology","institution_ids":["https://openalex.org/I12003341","https://openalex.org/I312958679"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Khan, Hassaan Muhammad","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I12003341","display_name":"National University of Science and Technology","ror":"https://ror.org/019vsm959","country_code":"RU","type":"education","lineage":["https://openalex.org/I12003341"]},{"id":"https://openalex.org/I312958679","display_name":"National University of Science and Technology","ror":"https://ror.org/02kesvt12","country_code":"ZW","type":"education","lineage":["https://openalex.org/I312958679"]}],"countries":["RU","ZW"],"is_corresponding":false,"raw_author_name":"Khan, Hassaan Muhammad","raw_affiliation_strings":["National University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Science and Technology","institution_ids":["https://openalex.org/I12003341","https://openalex.org/I312958679"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093598411","display_name":"Sondos Mahmoud Bsharat","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Bsharat, Sondos Mahmoud","raw_affiliation_strings":["McGill University \u266e"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"McGill University \u266e","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066530136","display_name":"Zhiqiang Shen","orcid":"https://orcid.org/0000-0002-4560-5092","observed_orcids":["https://orcid.org/0000-0002-4560-5092"]},"institutions":[{"id":"https://openalex.org/I12003341","display_name":"National University of Science and Technology","ror":"https://ror.org/019vsm959","country_code":"RU","type":"education","lineage":["https://openalex.org/I12003341"]},{"id":"https://openalex.org/I312958679","display_name":"National University of Science and Technology","ror":"https://ror.org/02kesvt12","country_code":"ZW","type":"education","lineage":["https://openalex.org/I312958679"]}],"countries":["RU","ZW"],"is_corresponding":false,"raw_author_name":"Shen, Zhiqiang","raw_affiliation_strings":["National University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Science and Technology","institution_ids":["https://openalex.org/I12003341","https://openalex.org/I312958679"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9740999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9740999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.011800000444054604,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.00430000014603138,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/retrieval-augmented-generation","display_name":"retrieval-augmented generation","score":0.9810000061988831},{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":0.9869999885559082},{"id":"https://openalex.org/keywords/small-language-models","display_name":"small language models","score":0.9359999895095825},{"id":"https://openalex.org/keywords/hallucination-mitigation","display_name":"hallucination mitigation","score":0.9100000262260437},{"id":"https://openalex.org/keywords/knowledge-graphs","display_name":"knowledge graphs","score":0.6970000267028809},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.5600000023841858},{"id":"https://openalex.org/keywords/user-privacy","display_name":"user privacy","score":0.7269999980926514},{"id":"https://openalex.org/keywords/factual-accuracy","display_name":"factual accuracy","score":0.6539999842643738}],"concepts":[{"id":"https://openalex.org/C2911011789","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Hallucinating","level":2,"score":0.8554999828338623},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7487000226974487},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5374000072479248},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5094000101089478},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4465999901294708},{"id":"https://openalex.org/C2987255567","wikidata":"https://www.wikidata.org/wiki/Q33002955","display_name":"Knowledge graph","level":2,"score":0.3889000117778778},{"id":"https://openalex.org/C2776960227","wikidata":"https://www.wikidata.org/wiki/Q2586354","display_name":"Knowledge transfer","level":2,"score":0.38370001316070557},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.367000013589859},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.3603000044822693},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.35370001196861267},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3301999866962433},{"id":"https://openalex.org/C2779458634","wikidata":"https://www.wikidata.org/wiki/Q24963715","display_name":"Debiasing","level":2,"score":0.3276999890804291},{"id":"https://openalex.org/C161301231","wikidata":"https://www.wikidata.org/wiki/Q3478658","display_name":"Knowledge representation and reasoning","level":2,"score":0.2994999885559082},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.28999999165534973},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.27730000019073486},{"id":"https://openalex.org/C115925183","wikidata":"https://www.wikidata.org/wiki/Q1412694","display_name":"Knowledge-based systems","level":2,"score":0.272599995136261},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2646999955177307},{"id":"https://openalex.org/C75553542","wikidata":"https://www.wikidata.org/wiki/Q178161","display_name":"A priori and a posteriori","level":2,"score":0.2565000057220459},{"id":"https://openalex.org/C2986930172","wikidata":"https://www.wikidata.org/wiki/Q1116133","display_name":"Social knowledge","level":2,"score":0.2526000142097473}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2506.01954","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.01954","pdf_url":"https://arxiv.org/pdf/2506.01954","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2506.01954","doi":"https://doi.org/10.48550/arxiv.2506.01954","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2506.01954","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2506.01954","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.01954","pdf_url":"https://arxiv.org/pdf/2506.01954","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4414899016.pdf","grobid_xml":"https://content.openalex.org/works/W4414899016.grobid-xml"},"referenced_works_count":27,"referenced_works":["https://openalex.org/W2794325560","https://openalex.org/W3083410900","https://openalex.org/W3111425437","https://openalex.org/W4386148138","https://openalex.org/W4387390366","https://openalex.org/W4387800173","https://openalex.org/W4391766322","https://openalex.org/W4392679398","https://openalex.org/W4395474395","https://openalex.org/W4395686609","https://openalex.org/W4399253046","https://openalex.org/W4399598106","https://openalex.org/W4399991263","https://openalex.org/W4400718844","https://openalex.org/W4401306886","https://openalex.org/W4401306979","https://openalex.org/W4404330874","https://openalex.org/W4405307166","https://openalex.org/W4405655184","https://openalex.org/W4405903187","https://openalex.org/W4408061739","https://openalex.org/W4410705984","https://openalex.org/W4411119264","https://openalex.org/W4411120544","https://openalex.org/W4415800738","https://openalex.org/W7126416899","https://openalex.org/W7166806268"],"related_works":["https://openalex.org/W4412888222","https://openalex.org/W4414692867","https://openalex.org/W4416139362","https://openalex.org/W7129352884","https://openalex.org/W7084448267","https://openalex.org/W7204572452","https://openalex.org/W7124936932","https://openalex.org/W4410887058","https://openalex.org/W6929235096","https://openalex.org/W4403570046"],"abstract_inverted_index":{"Retrieval-Augmented":[0],"Generation":[1],"(RAG)":[2],"methods":[3,135],"have":[4],"proven":[5],"highly":[6],"effective":[7],"for":[8,43,138],"tasks":[9],"requiring":[10],"factual":[11,71,101],"consistency":[12],"and":[13,25,60,78,92,99,116,151,159,166],"robust":[14],"knowledge":[15,46,61,72,90],"retrieval.":[16],"However,":[17],"large-scale":[18,48],"RAG":[19,45,134],"systems":[20],"consume":[21],"significant":[22],"computational":[23,79],"resources":[24],"are":[26],"prone":[27],"to":[28,142,162],"generating":[29],"hallucinated":[30],"content":[31],"from":[32,47],"Humans.":[33],"In":[34],"this":[35],"work,":[36],"we":[37,155],"introduce":[38,117],"$\\texttt{DRAG}$,":[39,154],"a":[40,88,106,118,157],"novel":[41],"framework":[42,111],"distilling":[44],"Language":[49],"Models":[50],"(LLMs)":[51],"into":[52],"small":[53],"LMs":[54],"(SLMs).":[55],"Our":[56],"approach":[57],"leverages":[58],"evidence-":[59],"graph-based":[62],"distillation,":[63],"ensuring":[64],"that":[65,127],"the":[66,83,131,145],"distilled":[67],"model":[68,76],"retains":[69],"critical":[70],"while":[73],"significantly":[74],"reducing":[75],"size":[77],"cost.":[80],"By":[81],"aligning":[82],"smaller":[84],"model's":[85],"predictions":[86],"with":[87],"structured":[89],"graph":[91],"ranked":[93],"evidence,":[94],"$\\texttt{DRAG}$":[95],"effectively":[96],"mitigates":[97,112],"hallucinations":[98],"improves":[100],"accuracy.":[102],"We":[103],"further":[104],"present":[105],"case":[107],"demonstrating":[108],"how":[109],"our":[110,128],"user":[113],"privacy":[114],"risks":[115],"corresponding":[119],"benchmark.":[120],"Experimental":[121],"evaluations":[122],"on":[123],"multiple":[124],"benchmarks":[125],"demonstrate":[126],"method":[129],"outperforms":[130],"prior":[132],"competitive":[133],"like":[136],"MiniRAG":[137],"SLMs":[139],"by":[140],"up":[141],"27.7%":[143],"using":[144],"same":[146],"models,":[147],"preserving":[148],"high-level":[149],"efficiency":[150],"reliability.":[152],"With":[153],"provide":[156],"practical":[158],"resource-efficient":[160],"roadmap":[161],"deploying":[163],"enhanced":[164],"retrieval":[165],"generation":[167],"capabilities":[168],"in":[169],"small-sized":[170],"LLMs.":[171]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-10T00:00:00"}