{"id":"https://openalex.org/W3105234097","doi":"https://doi.org/10.18653/v1/2020.sustainlp-1.16","title":"Load What You Need: Smaller Versions of Mutililingual BERT","display_name":"Load What You Need: Smaller Versions of Mutililingual BERT","publication_year":2020,"publication_date":"2020-01-01","ids":{"openalex":"https://openalex.org/W3105234097","doi":"https://doi.org/10.18653/v1/2020.sustainlp-1.16","mag":"3105234097"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2020.sustainlp-1.16","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2020.sustainlp-1.16","pdf_url":"https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of SustaiNLP: Workshop on Simple and Efficient Natural Language Processing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5066771882","display_name":"Amine Abdaoui","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164289","display_name":"G\u00e9osciences Environnement Toulouse","ror":"https://ror.org/05k0qmv73","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1327553481","https://openalex.org/I154202486","https://openalex.org/I2799535048","https://openalex.org/I2799535048","https://openalex.org/I2802818602","https://openalex.org/I4210088668","https://openalex.org/I4210090127","https://openalex.org/I4210113730","https://openalex.org/I4210131494","https://openalex.org/I4210148025","https://openalex.org/I4210164289","https://openalex.org/I4210166444","https://openalex.org/I4405257220","https://openalex.org/I4405258862"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Amine Abdaoui","raw_affiliation_strings":["Geotrend / Toulouse"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Geotrend / Toulouse","institution_ids":["https://openalex.org/I4210164289"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058662559","display_name":"Camille Pradel","orcid":"https://orcid.org/0000-0001-9944-563X"},"institutions":[{"id":"https://openalex.org/I4210164289","display_name":"G\u00e9osciences Environnement Toulouse","ror":"https://ror.org/05k0qmv73","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1327553481","https://openalex.org/I154202486","https://openalex.org/I2799535048","https://openalex.org/I2799535048","https://openalex.org/I2802818602","https://openalex.org/I4210088668","https://openalex.org/I4210090127","https://openalex.org/I4210113730","https://openalex.org/I4210131494","https://openalex.org/I4210148025","https://openalex.org/I4210164289","https://openalex.org/I4210166444","https://openalex.org/I4405257220","https://openalex.org/I4405258862"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Camille Pradel","raw_affiliation_strings":["Geotrend / Toulouse"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Geotrend / Toulouse","institution_ids":["https://openalex.org/I4210164289"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5033738221","display_name":"Gr\u00e9goire Sigel","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164289","display_name":"G\u00e9osciences Environnement Toulouse","ror":"https://ror.org/05k0qmv73","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1327553481","https://openalex.org/I154202486","https://openalex.org/I2799535048","https://openalex.org/I2799535048","https://openalex.org/I2802818602","https://openalex.org/I4210088668","https://openalex.org/I4210090127","https://openalex.org/I4210113730","https://openalex.org/I4210131494","https://openalex.org/I4210148025","https://openalex.org/I4210164289","https://openalex.org/I4210166444","https://openalex.org/I4405257220","https://openalex.org/I4405258862"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Gr\u00e9goire Sigel","raw_affiliation_strings":["Geotrend / Toulouse"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Geotrend / Toulouse","institution_ids":["https://openalex.org/I4210164289"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210164289"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":49,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"119","last_page":"123"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9790999889373779,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8201147317886353},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.7385144233703613},{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.5411268472671509},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.5074521899223328},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4513026475906372},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.422366201877594},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3821849226951599},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.0933157205581665},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.0748785138130188}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8201147317886353},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.7385144233703613},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.5411268472671509},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.5074521899223328},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4513026475906372},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.422366201877594},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3821849226951599},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0933157205581665},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.0748785138130188},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2020.sustainlp-1.16","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2020.sustainlp-1.16","pdf_url":"https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of SustaiNLP: Workshop on Simple and Efficient Natural Language Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2020.sustainlp-1.16","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2020.sustainlp-1.16","pdf_url":"https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of SustaiNLP: Workshop on Simple and Efficient Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.75,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3105234097.pdf","grobid_xml":"https://content.openalex.org/works/W3105234097.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W2294370754","https://openalex.org/W2891555348","https://openalex.org/W2896457183","https://openalex.org/W2924902521","https://openalex.org/W2963341956","https://openalex.org/W2963846996","https://openalex.org/W2970454332","https://openalex.org/W2970557265","https://openalex.org/W2976132230","https://openalex.org/W2978017171","https://openalex.org/W2980282514","https://openalex.org/W2990215755","https://openalex.org/W2996124573","https://openalex.org/W2998183051","https://openalex.org/W3035390927","https://openalex.org/W4288102732","https://openalex.org/W4288256350"],"related_works":["https://openalex.org/W3196747313","https://openalex.org/W3097571385","https://openalex.org/W2359001871","https://openalex.org/W3033862527","https://openalex.org/W4287761227","https://openalex.org/W1986205833","https://openalex.org/W3033942572","https://openalex.org/W3008110149","https://openalex.org/W4381786178","https://openalex.org/W4316012698"],"abstract_inverted_index":{"Pre-trained":[0],"Transformer-based":[1],"models":[2,21,73,122,141,174],"are":[3,43,177],"achieving":[4],"state-of-the-art":[5],"results":[6,115],"on":[7,59,95,167],"a":[8,24,158],"variety":[9],"of":[10,19,36,40,63,78,89,92,132,136,147],"Natural":[11],"Language":[12],"Processing":[13],"data":[14,98,170],"sets.":[15],"However,":[16],"the":[17,34,41,46,51,60,82,96,133,164,168],"size":[18,53],"these":[20],"is":[22],"often":[23],"drawback":[25],"for":[26],"their":[27],"deployment":[28],"in":[29,45,163],"real":[30],"production":[31],"applications.":[32],"In":[33,65],"case":[35],"multilingual":[37,93,111,148],"models,":[38],"most":[39],"parameters":[42],"located":[44],"embeddings":[47],"layer.":[48],"Therefore,":[49],"reducing":[50,128],"vocabulary":[52],"should":[54],"have":[55],"an":[56,87],"important":[57],"impact":[58],"total":[61,134],"number":[62,77,135],"parameters.":[64,137],"this":[66,104],"paper,":[67],"we":[68,101,118],"propose":[69],"to":[70,81,109,130,160],"generate":[71,120],"smaller":[72,90,121],"that":[74,103,117,123,152],"handle":[75],"fewer":[76],"languages":[79],"according":[80],"targeted":[83],"corpora.":[84],"We":[85,138],"present":[86],"evaluation":[88],"versions":[91],"BERT":[94],"XNLI":[97,169],"set,":[99],"but":[100],"believe":[102],"method":[105],"may":[106],"be":[107],"applied":[108],"other":[110],"transformers.":[112],"The":[113,172],"obtained":[114],"confirm":[116],"can":[119],"keep":[124],"comparable":[125],"results,":[126],"while":[127],"up":[129],"45%":[131],"compared":[139],"our":[140],"with":[142],"DistilmBERT":[143],"(a":[144],"distilled":[145],"version":[146],"BERT)":[149],"and":[150,175],"showed":[151],"unlike":[153],"language":[154],"reduction,":[155],"distillation":[156],"induced":[157],"1.7%":[159],"6%":[161],"drop":[162],"overall":[165],"accuracy":[166],"set.":[171],"presented":[173],"code":[176],"publicly":[178],"available.":[179]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":14},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":8}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}