{"id":"https://openalex.org/W4391244633","doi":"https://doi.org/10.48550/arxiv.2401.13303","title":"MaLA-500: Massive Language Adaptation of Large Language Models","display_name":"MaLA-500: Massive Language Adaptation of Large Language Models","publication_year":2024,"publication_date":"2024-01-24","ids":{"openalex":"https://openalex.org/W4391244633","doi":"https://doi.org/10.48550/arxiv.2401.13303"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2401.13303","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.13303","pdf_url":"https://arxiv.org/pdf/2401.13303","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2401.13303","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014957657","display_name":"Peiqin Lin","orcid":"https://orcid.org/0000-0003-2818-3008","observed_orcids":["https://orcid.org/0000-0003-2818-3008"]},"institutions":[{"id":"https://openalex.org/I4403386549","display_name":"Munich Center for Machine Learning","ror":"https://ror.org/02nfy3535","country_code":"DE","type":"education","lineage":["https://openalex.org/I4403386549","https://openalex.org/I62916508","https://openalex.org/I8204097"]},{"id":"https://openalex.org/I8204097","display_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","ror":"https://ror.org/05591te55","country_code":"DE","type":"education","lineage":["https://openalex.org/I8204097"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Lin, Peiqin","raw_affiliation_strings":["Center for Information and Language Processing , LMU Munich","Munich Center for Machine Learning"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Information and Language Processing , LMU Munich","institution_ids":["https://openalex.org/I8204097"]},{"raw_affiliation_string":"Munich Center for Machine Learning","institution_ids":["https://openalex.org/I4403386549"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007841686","display_name":"Shaoxiong Ji","orcid":"https://orcid.org/0000-0003-3281-8002","observed_orcids":["https://orcid.org/0000-0003-3281-8002"]},"institutions":[{"id":"https://openalex.org/I133731052","display_name":"University of Helsinki","ror":"https://ror.org/040af2s02","country_code":"FI","type":"education","lineage":["https://openalex.org/I133731052"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Ji, Shaoxiong","raw_affiliation_strings":["University of Helsinki"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Helsinki","institution_ids":["https://openalex.org/I133731052"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082417280","display_name":"J\u00f6rg Tiedemann","orcid":"https://orcid.org/0000-0003-3065-7989","observed_orcids":["https://orcid.org/0000-0003-3065-7989"]},"institutions":[{"id":"https://openalex.org/I133731052","display_name":"University of Helsinki","ror":"https://ror.org/040af2s02","country_code":"FI","type":"education","lineage":["https://openalex.org/I133731052"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Tiedemann, J\u00f6rg","raw_affiliation_strings":["University of Helsinki"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Helsinki","institution_ids":["https://openalex.org/I133731052"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051693368","display_name":"Andr\u00e9 F. T. Martins","orcid":"https://orcid.org/0000-0001-8282-625X","observed_orcids":["https://orcid.org/0000-0001-8282-625X","https://orcid.org/0000-0001-6863-4681"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Martins, Andr\u00e9 F. T.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5071144367","display_name":"Hinrich Sch\u00fctze","orcid":"https://orcid.org/0000-0001-9514-7934","observed_orcids":["https://orcid.org/0000-0001-9514-7934"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sch\u00fctze, Hinrich","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5214999914169312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5214999914169312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.461899995803833,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.007699999958276749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multilingual-large-language-models","display_name":"multilingual large language models","score":0.9409999847412109},{"id":"https://openalex.org/keywords/low-resource-languages","display_name":"low-resource languages","score":0.9810000061988831},{"id":"https://openalex.org/keywords/vocabulary-expansion","display_name":"vocabulary enrichment","score":0.6759999990463257},{"id":"https://openalex.org/keywords/continual-pre-training","display_name":"continual pre-training","score":0.8169999718666077},{"id":"https://openalex.org/keywords/llama-2","display_name":"Llama 2","score":0.8970000147819519},{"id":"https://openalex.org/keywords/in-context-learning","display_name":"in-context learning","score":0.7620000243186951},{"id":"https://openalex.org/keywords/text-generation","display_name":"text generation","score":0.41200000047683716}],"concepts":[{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.6382726430892944},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6176241040229797},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.5851942300796509},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.5800437927246094},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5712168216705322},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.5135200619697571},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.45902687311172485},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.44378143548965454},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4260137677192688},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3870021104812622},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.36732104420661926},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3384981155395508},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.25979870557785034},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.10501551628112793},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.06674656271934509},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C105702510","wikidata":"https://www.wikidata.org/wiki/Q514","display_name":"Anatomy","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2401.13303","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.13303","pdf_url":"https://arxiv.org/pdf/2401.13303","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2401.13303","doi":"https://doi.org/10.48550/arxiv.2401.13303","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2401.13303","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2401.13303","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.13303","pdf_url":"https://arxiv.org/pdf/2401.13303","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1783285269","display_name":"Deep Learning for Structured Prediction in Natural Language Processing","funder_award_id":"758969","funder_id":"https://openalex.org/F4320338335","funder_display_name":"H2020 European Research Council"},{"id":"https://openalex.org/G3689595285","display_name":"Found in Translation \u2013 Natural Language Understanding with Cross-Lingual Grounding","funder_award_id":"771113","funder_id":"https://openalex.org/F4320338335","funder_display_name":"H2020 European Research Council"},{"id":"https://openalex.org/G4364068006","display_name":"Non-sequence models for tokenization replacement","funder_award_id":"740516","funder_id":"https://openalex.org/F4320338335","funder_display_name":"H2020 European Research Council"},{"id":"https://openalex.org/G4459410915","display_name":null,"funder_award_id":"771113","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5112024542","display_name":"High Performance Language Technologies","funder_award_id":"101070350","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5298389108","display_name":"Unified Transcription and Translation for Extended Reality","funder_award_id":"101070631","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G612072080","display_name":null,"funder_award_id":"951847","funder_id":"https://openalex.org/F4320334322","funder_display_name":"HORIZON EUROPE Framework Programme"},{"id":"https://openalex.org/G6251754251","display_name":"European Learning and Intelligent Systems Excellence","funder_award_id":"951847","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G7901452417","display_name":"Deep Learning for Structured Prediction in Natural Language Processing","funder_award_id":"758969","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320314707","display_name":"Government of the United Kingdom","ror":"https://ror.org/05wnh3t63"},{"id":"https://openalex.org/F4320314731","display_name":"UK Research and Innovation","ror":"https://ror.org/001aqnf71"},{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320322725","display_name":"China Scholarship Council","ror":"https://ror.org/04atp4p48"},{"id":"https://openalex.org/F4320334322","display_name":"HORIZON EUROPE Framework Programme","ror":null},{"id":"https://openalex.org/F4320338335","display_name":"H2020 European Research Council","ror":"https://ror.org/0472cxd90"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391244633.pdf","grobid_xml":"https://content.openalex.org/works/W4391244633.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4405094822","https://openalex.org/W4403796797","https://openalex.org/W4396913951","https://openalex.org/W4404792854","https://openalex.org/W4416010939","https://openalex.org/W4402669950","https://openalex.org/W7163905643","https://openalex.org/W7092285609","https://openalex.org/W4394708800","https://openalex.org/W7140158674"],"abstract_inverted_index":{"Large":[0],"language":[1,13,47],"models":[2],"(LLMs)":[3],"have":[4],"advanced":[5],"the":[6,9,83,94],"state":[7],"of":[8,25,55,86,97],"art":[10],"in":[11,31],"natural":[12],"processing.":[14],"However,":[15],"their":[16,32],"predominant":[17],"design":[18],"for":[19,34],"English":[20],"or":[21],"a":[22,28,44,111],"limited":[23],"set":[24],"languages":[26,88],"creates":[27],"substantial":[29],"gap":[30],"effectiveness":[33],"low-resource":[35,87],"languages.":[36,57,121],"To":[37,58],"bridge":[38],"this":[39],"gap,":[40],"we":[41,61],"introduce":[42],"MaLA-500,":[43,60],"novel":[45],"large":[46],"model":[48],"designed":[49],"to":[50],"cover":[51],"an":[52],"extensive":[53],"range":[54],"534":[56],"train":[59],"employ":[62],"vocabulary":[63],"extension":[64],"and":[65,108,116],"continued":[66],"pretraining":[67],"on":[68,106],"LLaMA":[69],"2":[70],"with":[71],"Glot500-c.":[72],"Our":[73],"intrinsic":[74],"evaluation":[75,96],"demonstrates":[76],"that":[77,101],"MaLA-500":[78,102,124],"is":[79],"better":[80],"at":[81,125],"predicting":[82],"given":[84],"texts":[85],"than":[89],"existing":[90],"multilingual":[91],"LLMs.":[92],"Moreover,":[93],"extrinsic":[95],"in-context":[98],"learning":[99],"shows":[100],"outperforms":[103],"previous":[104],"LLMs":[105],"SIB200":[107],"Taxi1500":[109],"by":[110],"significant":[112],"margin,":[113],"i.e.,":[114],"11.68%":[115],"4.82%":[117],"marco-average":[118],"accuracy":[119],"across":[120],"We":[122],"release":[123],"https://huggingface.co/MaLA-LM":[126]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-10-27T15:51:10.671819","created_date":"2024-01-26T00:00:00"}