{"id":"https://openalex.org/W4214488463","doi":"https://doi.org/10.1109/taslp.2022.3153264","title":"Multi-Teacher Distillation With Single Model for Neural Machine Translation","display_name":"Multi-Teacher Distillation With Single Model for Neural Machine Translation","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4214488463","doi":"https://doi.org/10.1109/taslp.2022.3153264"},"language":"en","primary_location":{"id":"doi:10.1109/taslp.2022.3153264","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2022.3153264","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102023923","display_name":"Xiaobo Liang","orcid":"https://orcid.org/0009-0001-1550-2877"},"institutions":[{"id":"https://openalex.org/I3923682","display_name":"Soochow University","ror":"https://ror.org/05t8y2r12","country_code":"CN","type":"education","lineage":["https://openalex.org/I3923682"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xiaobo Liang","raw_affiliation_strings":["Soochow University, Suzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Soochow University, Suzhou, China","institution_ids":["https://openalex.org/I3923682"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102750692","display_name":"Lijun Wu","orcid":"https://orcid.org/0000-0002-3530-590X"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lijun Wu","raw_affiliation_strings":["Microsoft Research, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-3530-590X","affiliations":[{"raw_affiliation_string":"Microsoft Research, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100657514","display_name":"Juntao Li","orcid":"https://orcid.org/0000-0002-6286-7529"},"institutions":[{"id":"https://openalex.org/I3923682","display_name":"Soochow University","ror":"https://ror.org/05t8y2r12","country_code":"CN","type":"education","lineage":["https://openalex.org/I3923682"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Juntao Li","raw_affiliation_strings":["Soochow University, Suzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-6286-7529","affiliations":[{"raw_affiliation_string":"Soochow University, Suzhou, China","institution_ids":["https://openalex.org/I3923682"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020025718","display_name":"Tao Qin","orcid":"https://orcid.org/0000-0002-9095-0776"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tao Qin","raw_affiliation_strings":["Microsoft Research, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9095-0776","affiliations":[{"raw_affiliation_string":"Microsoft Research, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100402911","display_name":"Min Zhang","orcid":"https://orcid.org/0000-0002-3895-5510"},"institutions":[{"id":"https://openalex.org/I3923682","display_name":"Soochow University","ror":"https://ror.org/05t8y2r12","country_code":"CN","type":"education","lineage":["https://openalex.org/I3923682"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Zhang","raw_affiliation_strings":["Soochow University, Suzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Soochow University, Suzhou, China","institution_ids":["https://openalex.org/I3923682"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101884287","display_name":"Tie\u2010Yan Liu","orcid":"https://orcid.org/0000-0002-0476-8020"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tie-Yan Liu","raw_affiliation_strings":["Microsoft Research, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5102023923"],"corresponding_institution_ids":["https://openalex.org/I3923682"],"apc_list":null,"apc_paid":null,"fwci":2.6359,"has_fulltext":false,"cited_by_count":21,"citation_normalized_percentile":{"value":0.90983462,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"30","issue":null,"first_page":"992","last_page":"1002"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6850056648254395},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5874069929122925},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5782567858695984},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5407431721687317},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4537144899368286},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.334434449672699},{"id":"https://openalex.org/keywords/chromatography","display_name":"Chromatography","score":0.07026275992393494}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6850056648254395},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5874069929122925},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5782567858695984},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5407431721687317},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4537144899368286},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.334434449672699},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.07026275992393494},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taslp.2022.3153264","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2022.3153264","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.8399999737739563}],"awards":[{"id":"https://openalex.org/G5610393340","display_name":null,"funder_award_id":"62036004","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W2095705004","https://openalex.org/W2105482032","https://openalex.org/W2294370754","https://openalex.org/W2889326796","https://openalex.org/W2937408455","https://openalex.org/W2944869267","https://openalex.org/W2945052683","https://openalex.org/W2962784628","https://openalex.org/W2963206148","https://openalex.org/W2963736842","https://openalex.org/W2963807318","https://openalex.org/W2963878748","https://openalex.org/W2963925437","https://openalex.org/W2983128379","https://openalex.org/W2988975212","https://openalex.org/W2996834012","https://openalex.org/W3034906024","https://openalex.org/W3034949522","https://openalex.org/W3035691519","https://openalex.org/W3036939249","https://openalex.org/W3037585139","https://openalex.org/W3046158644","https://openalex.org/W3092173171","https://openalex.org/W3101005014","https://openalex.org/W3116310032","https://openalex.org/W3155270181","https://openalex.org/W3172096628","https://openalex.org/W3173982024","https://openalex.org/W4298159529","https://openalex.org/W6631190155","https://openalex.org/W6638523607","https://openalex.org/W6674330103","https://openalex.org/W6679436768","https://openalex.org/W6679909955","https://openalex.org/W6684488266","https://openalex.org/W6732837357","https://openalex.org/W6733321041","https://openalex.org/W6737778391","https://openalex.org/W6739901393","https://openalex.org/W6748641889","https://openalex.org/W6751979845","https://openalex.org/W6757249106","https://openalex.org/W6757585730","https://openalex.org/W6757736129","https://openalex.org/W6758657797","https://openalex.org/W6761205521","https://openalex.org/W6763608318","https://openalex.org/W6768080748","https://openalex.org/W6769921281","https://openalex.org/W6774062504","https://openalex.org/W6780021571","https://openalex.org/W6780039898","https://openalex.org/W6783432658","https://openalex.org/W6789529132"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3107602296","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"Knowledge":[0],"distillation":[1,96],"(KD)":[2],"is":[3,60,70,212],"an":[4],"effective":[5],"strategy":[6],"for":[7],"neural":[8],"machine":[9],"translation":[10,185],"(NMT)":[11],"to":[12,27,61,92],"improve":[13],"the":[14,21,25,32,39,43,72,82,98,177],"performance":[15,195],"of":[16,84,104],"a":[17,57,112],"student":[18,26],"model.":[19,55,108],"Usually,":[20],"teacher":[22,40,47,54,65,85,95,107,113,129,198],"can":[23,131],"guide":[24],"be":[28],"better":[29],"by":[30,114],"distilling":[31],"soft":[33],"label":[34],"or":[35],"data":[36,44],"knowledge":[37,48,208],"from":[38,97],"itself.":[41],"However,":[42],"diversity":[45],"and":[46,75,101,122,136,164,176,206],"are":[49,78],"limited":[50],"with":[51,81,196,200],"only":[52],"one":[53,67,105,128],"Though":[56],"natural":[58],"solution":[59],"adopt":[62],"multiple":[63,94,115,133,197],"randomized":[64,202],"models,":[66],"big":[68],"shortcoming":[69],"that":[71,188],"model":[73,130],"parameters":[74,140],"training":[76,144],"costs":[77],"largely":[79],"increased":[80],"number":[83],"models.":[86],"In":[87,125],"this":[88],"work,":[89],"we":[90,110],"explore":[91],"mimic":[93],"sub-network":[99,116],"space":[100],"permuted":[102],"variants":[103,135],"single":[106],"Specifically,":[109],"train":[111],"extraction":[117],"paradigms:":[118],"sub-layer":[119],"reordering,":[120],"layer-drop,":[121],"dropout":[123],"variants.":[124],"doing":[126],"so,":[127],"provide":[132],"outputs":[134],"causes":[137],"neither":[138],"additional":[139],"nor":[141],"much":[142],"extra":[143],"cost.":[145],"Experiments":[146],"on":[147],"<formula":[148,155,160,167,172,181],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[149,156,161,168,173,182],"xmlns:xlink=\"http://www.w3.org/1999/xlink\"><tex>$8$</tex></formula>":[150],"IWSLT":[151],"datasets:":[152],"(IWSLT14":[153],"En":[154,159,166,171],"xmlns:xlink=\"http://www.w3.org/1999/xlink\"><tex>$\\leftrightarrow$</tex></formula>":[157,162,169,174],"De,":[158],"Es,":[163],"IWSLT17":[165],"Fr,":[170],"Zh)":[175],"large":[178],"WMT14":[179],"EN":[180],"xmlns:xlink=\"http://www.w3.org/1999/xlink\"><tex>$\\to$</tex></formula>":[183],"DE":[184],"tasks":[186],"show":[187],"our":[189],"method":[190],"even":[191],"achieves":[192],"nearly":[193],"comparable":[194],"models":[199],"different":[201],"parameters,":[203],"both":[204],"word-level,":[205],"sequence-level":[207],"distillation.":[209],"Our":[210],"code":[211],"available":[213],"at":[214],"GitHub\\footnote{https://github.com/dropreg/RLD}":[215]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}