{"id":"https://openalex.org/W3101045333","doi":"https://doi.org/10.48550/arxiv.2002.10957","title":"MiniLM: Deep Self-Attention Distillation for Task-Agnostic Compression of Pre-Trained Transformers","display_name":"MiniLM: Deep Self-Attention Distillation for Task-Agnostic Compression of Pre-Trained Transformers","publication_year":2020,"publication_date":"2020-02-25","ids":{"openalex":"https://openalex.org/W3101045333","doi":"https://doi.org/10.48550/arxiv.2002.10957","mag":"3101045333"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2002.10957","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2002.10957","pdf_url":"https://arxiv.org/pdf/2002.10957","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2002.10957","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100370079","display_name":"Wenhui Wang","orcid":"https://orcid.org/0000-0002-9623-0489","observed_orcids":["https://orcid.org/0000-0002-9623-0489"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Wenhui","raw_affiliation_strings":["Microsoft Research","Microsoft (United States), Redmond, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft (United States), Redmond, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014662947","display_name":"Furu Wei","orcid":"https://orcid.org/0000-0002-7810-5852","observed_orcids":["https://orcid.org/0000-0002-7810-5852"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wei, Furu","raw_affiliation_strings":["Microsoft Research","Microsoft (United States), Redmond, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft (United States), Redmond, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101751776","display_name":"Li Dong","orcid":"https://orcid.org/0000-0003-2036-1009","observed_orcids":["https://orcid.org/0000-0003-2036-1009"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dong, Li","raw_affiliation_strings":["Microsoft (United States), Redmond, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft (United States), Redmond, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053932343","display_name":"Hangbo Bao","orcid":"https://orcid.org/0000-0002-5773-8781","observed_orcids":["https://orcid.org/0000-0002-5773-8781"]},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bao, Hangbo","raw_affiliation_strings":["Harbin Institute of Technology, Harbin, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Harbin, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072808499","display_name":"Nan Yang","orcid":"https://orcid.org/0000-0002-2621-8927","observed_orcids":["https://orcid.org/0000-0002-2621-8927"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang, Nan","raw_affiliation_strings":["Microsoft (United States), Redmond, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft (United States), Redmond, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100701572","display_name":"Ming Zhou","orcid":"https://orcid.org/0000-0002-2551-2964","observed_orcids":["https://orcid.org/0000-0002-2551-2964"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhou, Ming","raw_affiliation_strings":["Microsoft Research","Microsoft (United States), Redmond, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft (United States), Redmond, United States","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1085,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"33","issue":null,"first_page":"5776","last_page":"5788"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5303000211715698,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5303000211715698,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.4253000020980835,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.030300000682473183,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":0.9120000004768372},{"id":"https://openalex.org/keywords/transformer-compression","display_name":"transformer compression","score":0.6549999713897705},{"id":"https://openalex.org/keywords/pre-trained-language-models","display_name":"pre-trained language models","score":0.8539999723434448},{"id":"https://openalex.org/keywords/teacher-assistants","display_name":"teacher assistants","score":0.5440000295639038},{"id":"https://openalex.org/keywords/scaled-dot-product-attention","display_name":"scaled dot-product attention","score":0.7609999775886536},{"id":"https://openalex.org/keywords/parameter-efficient-models","display_name":"parameter-efficient models","score":0.5270000100135803},{"id":"https://openalex.org/keywords/bert","display_name":"BERT","score":0.6869999766349792},{"id":"https://openalex.org/keywords/multilingual-models","display_name":"multilingual models","score":0.7120000123977661},{"id":"https://openalex.org/keywords/squad","display_name":"SQuAD","score":0.6039999723434448},{"id":"https://openalex.org/keywords/glue-benchmark","display_name":"GLUE benchmark","score":0.8849999904632568}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.8234264850616455},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7803620100021362},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5480260848999023},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5234108567237854},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5155925154685974},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5049148201942444},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.48459160327911377},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.46313905715942383},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43232691287994385},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.16549456119537354},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.09071248769760132},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.07850092649459839},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0770125687122345},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2002.10957","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2002.10957","pdf_url":"https://arxiv.org/pdf/2002.10957","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2002.10957","doi":"https://doi.org/10.48550/arxiv.2002.10957","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2002.10957","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:3101045333","doi":null,"is_oa":false,"landing_page_url":"https://proceedings.neurips.cc/paper/2020/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"Neural Information Processing Systems","raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2002.10957","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2002.10957","pdf_url":"https://arxiv.org/pdf/2002.10957","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3101045333.pdf","grobid_xml":"https://content.openalex.org/works/W3101045333.grobid-xml"},"referenced_works_count":58,"referenced_works":["https://openalex.org/W131533222","https://openalex.org/W1566289585","https://openalex.org/W1599016936","https://openalex.org/W1690739335","https://openalex.org/W1821462560","https://openalex.org/W2130158090","https://openalex.org/W2154652894","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2251939518","https://openalex.org/W2396767181","https://openalex.org/W2525127255","https://openalex.org/W2525778437","https://openalex.org/W2561238782","https://openalex.org/W2606974598","https://openalex.org/W2607892599","https://openalex.org/W2626778328","https://openalex.org/W2739351760","https://openalex.org/W2787560479","https://openalex.org/W2805206884","https://openalex.org/W2806120502","https://openalex.org/W2888482885","https://openalex.org/W2889518897","https://openalex.org/W2890166583","https://openalex.org/W2891555348","https://openalex.org/W2920812691","https://openalex.org/W2924902521","https://openalex.org/W2944815030","https://openalex.org/W2945260553","https://openalex.org/W2948947170","https://openalex.org/W2950813464","https://openalex.org/W2953109491","https://openalex.org/W2953337107","https://openalex.org/W2962753370","https://openalex.org/W2963310665","https://openalex.org/W2963323070","https://openalex.org/W2963341956","https://openalex.org/W2963350559","https://openalex.org/W2963748441","https://openalex.org/W2964121744","https://openalex.org/W2964308564","https://openalex.org/W2965373594","https://openalex.org/W2969515962","https://openalex.org/W2969601108","https://openalex.org/W2970049541","https://openalex.org/W2970419734","https://openalex.org/W2970557265","https://openalex.org/W2972437349","https://openalex.org/W2975429091","https://openalex.org/W2978017171","https://openalex.org/W2980404057","https://openalex.org/W2981852735","https://openalex.org/W2982399380","https://openalex.org/W2983040767","https://openalex.org/W2996569511","https://openalex.org/W2997666887","https://openalex.org/W3007759824","https://openalex.org/W3105966348"],"related_works":["https://openalex.org/W3174544005","https://openalex.org/W3166068809","https://openalex.org/W3169008558","https://openalex.org/W2975429091","https://openalex.org/W4402167571","https://openalex.org/W4283789538","https://openalex.org/W3171155106","https://openalex.org/W4226075153","https://openalex.org/W4382119231","https://openalex.org/W7103754027"],"abstract_inverted_index":{"Pre-trained":[0],"language":[1],"models":[2,23],"(e.g.,":[3],"BERT":[4],"(Devlin":[5],"et":[6,64,171],"al.,":[7,65,172],"2018)":[8],"and":[9,37,46,56,118,153,211,222],"its":[10],"variants)":[11],"have":[12,156],"achieved":[13],"remarkable":[14],"success":[15],"in":[16,40,92,131,141,159,193,233],"varieties":[17],"of":[18,26,28,30,95,107,112,151,178,197,218,224],"NLP":[19],"tasks.":[20],"However,":[21],"these":[22],"usually":[24],"consist":[25],"hundreds":[27],"millions":[29],"parameters":[31,221],"which":[32,87,115],"brings":[33],"challenges":[34],"for":[35,120],"fine-tuning":[36],"online":[38],"serving":[39],"real-life":[41],"applications":[42],"due":[43],"to":[44,59,143,238],"latency":[45],"capacity":[47],"constraints.":[48],"In":[49,200],"this":[50],"work,":[51],"we":[52,101,124,163],"present":[53],"a":[54,89,167],"simple":[55],"effective":[57,117],"approach":[58],"compress":[60],"large":[61,97,179],"Transformer":[62,93,110,181,220],"(Vaswani":[63],"2017)":[66],"based":[67],"pre-trained":[68,180,240],"models,":[69],"termed":[70],"as":[71,135],"deep":[72,138,235],"self-attention":[73,85,105,133,139,236],"distillation.":[74],"The":[75],"small":[76],"model":[77,98,189],"(student)":[78],"is":[79,116],"trained":[80],"by":[81],"deeply":[82],"mimicking":[83],"the":[84,96,104,108,113,121,126,132,136,144,148,176,219,225],"module,":[86],"plays":[88],"vital":[90],"role":[91],"networks,":[94],"(teacher).":[99],"Specifically,":[100],"propose":[102],"distilling":[103],"module":[106,134],"last":[109],"layer":[111],"teacher,":[114],"flexible":[119],"student.":[122],"Furthermore,":[123],"introduce":[125],"scaled":[127,149],"dot-product":[128,150],"between":[129],"values":[130],"new":[137],"knowledge,":[140],"addition":[142],"attention":[145],"distributions":[146],"(i.e.,":[147],"queries":[152],"keys)":[154],"that":[155,165,186],"been":[157],"used":[158],"existing":[160],"works.":[161],"Moreover,":[162],"show":[164],"introducing":[166],"teacher":[168,226],"assistant":[169],"(Mirzadeh":[170],"2019)":[173],"also":[174,229],"helps":[175],"distillation":[177,237],"models.":[182,199,241],"Experimental":[183],"results":[184,232],"demonstrate":[185],"our":[187],"monolingual":[188],"outperforms":[190],"state-of-the-art":[191],"baselines":[192],"different":[194],"parameter":[195],"size":[196],"student":[198],"particular,":[201],"it":[202],"retains":[203],"more":[204],"than":[205],"99%":[206],"accuracy":[207],"on":[208],"SQuAD":[209],"2.0":[210],"several":[212],"GLUE":[213],"benchmark":[214],"tasks":[215],"using":[216],"50%":[217],"computations":[223],"model.":[227],"We":[228],"obtain":[230],"competitive":[231],"applying":[234],"multilingual":[239]},"counts_by_year":[{"year":2026,"cited_by_count":226},{"year":2025,"cited_by_count":201},{"year":2024,"cited_by_count":183},{"year":2023,"cited_by_count":229},{"year":2022,"cited_by_count":138},{"year":2021,"cited_by_count":94},{"year":2020,"cited_by_count":14}],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2020-11-23T00:00:00"}