{"id":"https://openalex.org/W4281492541","doi":"https://doi.org/10.18653/v1/2022.deeplo-1.10","title":"Pre-training Data Quality and Quantity for a Low-Resource Language: New Corpus and BERT Models for Maltese","display_name":"Pre-training Data Quality and Quantity for a Low-Resource Language: New Corpus and BERT Models for Maltese","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4281492541","doi":"https://doi.org/10.18653/v1/2022.deeplo-1.10"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2022.deeplo-1.10","doi":"https://doi.org/10.18653/v1/2022.deeplo-1.10","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.deeplo-1.10","pdf_url":"https://aclanthology.org/2022.deeplo-1.10.pdf","source":{"id":"https://openalex.org/S7407115056","display_name":"Workshop on Deep Learning Approaches for Low-Resource NLP (DeepLo)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Third Workshop on Deep Learning for Low-Resource Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2022.deeplo-1.10.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051863994","display_name":"Kurt Micallef","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I197854408","display_name":"University of Malta","ror":"https://ror.org/03a62bv60","country_code":"MT","type":"education","lineage":["https://openalex.org/I197854408"]}],"countries":["MT"],"is_corresponding":false,"raw_author_name":"Kurt Micallef","raw_affiliation_strings":["Department of Artificial Intelligence , University of Malta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Artificial Intelligence , University of Malta","institution_ids":["https://openalex.org/I197854408"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027041248","display_name":"Albert Gatt A","orcid":"https://orcid.org/0000-0001-6388-8244","observed_orcids":["https://orcid.org/0000-0001-6388-8244"]},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]},{"id":"https://openalex.org/I197854408","display_name":"University of Malta","ror":"https://ror.org/03a62bv60","country_code":"MT","type":"education","lineage":["https://openalex.org/I197854408"]}],"countries":["MT","NL"],"is_corresponding":false,"raw_author_name":"Albert Gatt","raw_affiliation_strings":["Information and Computing Sciences , Utrecht University","Institute of Linguistics and Language Technology , University of Malta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Information and Computing Sciences , Utrecht University","institution_ids":["https://openalex.org/I193662353"]},{"raw_affiliation_string":"Institute of Linguistics and Language Technology , University of Malta","institution_ids":["https://openalex.org/I197854408"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048549686","display_name":"Marc Tanti","orcid":"https://orcid.org/0000-0002-7156-0596","observed_orcids":["https://orcid.org/0000-0002-7156-0596"]},"institutions":[{"id":"https://openalex.org/I197854408","display_name":"University of Malta","ror":"https://ror.org/03a62bv60","country_code":"MT","type":"education","lineage":["https://openalex.org/I197854408"]}],"countries":["MT"],"is_corresponding":false,"raw_author_name":"Marc Tanti","raw_affiliation_strings":["Institute of Linguistics and Language Technology , University of Malta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Linguistics and Language Technology , University of Malta","institution_ids":["https://openalex.org/I197854408"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5022407073","display_name":"Lonneke van der Plas","orcid":"https://orcid.org/0000-0002-2871-2574","observed_orcids":["https://orcid.org/0000-0002-2871-2574"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lonneke van der Plas and Claudia Borg","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.659,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":{"value":0.86690194,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"90","last_page":"101"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5898000001907349,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5898000001907349,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4011000096797943,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.002899999963119626,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/maltese-language","display_name":"Maltese language","score":0.8870000243186951},{"id":"https://openalex.org/keywords/low-resource-languages","display_name":"low-resource languages","score":0.9739999771118164},{"id":"https://openalex.org/keywords/multilingual-bert","display_name":"multilingual BERT","score":0.6869999766349792},{"id":"https://openalex.org/keywords/dependency-parsing","display_name":"dependency parsing","score":0.7590000033378601},{"id":"https://openalex.org/keywords/part-of-speech-tagging","display_name":"part-of-speech tagging","score":1.0},{"id":"https://openalex.org/keywords/named-entity-recognition","display_name":"named entity recognition","score":0.9940000176429749},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"sentiment analysis","score":0.9950000047683716},{"id":"https://openalex.org/keywords/cross-lingual-transfer","display_name":"cross-lingual transfer","score":0.8029999732971191}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8255484104156494},{"id":"https://openalex.org/C2775911240","wikidata":"https://www.wikidata.org/wiki/Q9166","display_name":"Maltese","level":2,"score":0.8090602159500122},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.7027633786201477},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6135637760162354},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5831831693649292},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5556589365005493},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.5489261746406555},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.5076123476028442},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.49247726798057556},{"id":"https://openalex.org/C164883195","wikidata":"https://www.wikidata.org/wiki/Q674834","display_name":"Dependency grammar","level":3,"score":0.48982641100883484},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4384623169898987},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.43260687589645386},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.21912461519241333},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.18653/v1/2022.deeplo-1.10","doi":"https://doi.org/10.18653/v1/2022.deeplo-1.10","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.deeplo-1.10","pdf_url":"https://aclanthology.org/2022.deeplo-1.10.pdf","source":{"id":"https://openalex.org/S7407115056","display_name":"Workshop on Deep Learning Approaches for Low-Resource NLP (DeepLo)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Third Workshop on Deep Learning for Low-Resource Natural Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2205.10517","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.10517","pdf_url":"https://arxiv.org/pdf/2205.10517","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:dspace.library.uu.nl:1874/425952","doi":null,"is_oa":true,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/425952","pdf_url":null,"source":{"id":"https://openalex.org/S4306401649","display_name":"Utrecht University Repository (Utrecht University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I193662353","host_organization_name":"Utrecht University","host_organization_lineage":["https://openalex.org/I193662353"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Part of book"},{"id":"pmh:uu:oai:dspace.library.uu.nl:1874/425952","doi":null,"is_oa":true,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/425952","pdf_url":null,"source":{"id":"https://openalex.org/S4306401649","display_name":"Utrecht University Repository (Utrecht University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I193662353","host_organization_name":"Utrecht University","host_organization_lineage":["https://openalex.org/I193662353"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proceedings of the Third Workshop on Deep Learning for Low Resource NLP (DeepLo'22), 90. Association for Computational Linguistics","raw_type":"info:eu-repo/semantics/bookpart"},{"id":"doi:10.48550/arxiv.2205.10517","doi":"https://doi.org/10.48550/arxiv.2205.10517","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2205.10517","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"doi:10.18653/v1/2022.deeplo-1.10","doi":"https://doi.org/10.18653/v1/2022.deeplo-1.10","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.deeplo-1.10","pdf_url":"https://aclanthology.org/2022.deeplo-1.10.pdf","source":{"id":"https://openalex.org/S7407115056","display_name":"Workshop on Deep Learning Approaches for Low-Resource NLP (DeepLo)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Third Workshop on Deep Learning for Low-Resource Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320329157","display_name":"Universit\u00e0 ta' Malta","ror":"https://ror.org/03a62bv60"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4281492541.pdf","grobid_xml":"https://content.openalex.org/works/W4281492541.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7165413686","https://openalex.org/W4385572391","https://openalex.org/W4412887862","https://openalex.org/W3038047279","https://openalex.org/W7131904587","https://openalex.org/W4385573670","https://openalex.org/W7002318660","https://openalex.org/W7172452368","https://openalex.org/W3213418658","https://openalex.org/W3089959042"],"abstract_inverted_index":{"Multilingual":[0],"language":[1,38],"models":[2,59,142,162],"such":[3],"as":[4],"mBERT":[5,44],"have":[6,97],"seen":[7],"impressive":[8],"cross-lingual":[9],"transfer":[10],"to":[11,114,129],"a":[12,36,47,80,106,122,147,156],"variety":[13],"of":[14,30,49,108,124],"languages,":[15],"but":[16],"many":[17],"languages":[18],"remain":[19],"excluded":[20],"from":[21,152],"these":[22,167],"models.In":[23],"this":[24,125],"paper,":[25],"we":[26],"analyse":[27],"the":[28,56,88,91,99,144,170,192],"effect":[29,89],"pre-training":[31,50,92,109],"with":[32,55,189],"monolingual":[33,148],"data":[34,93],"for":[35,84,180,197],"low-resource":[37],"that":[39,90,104,121],"is":[40,111,127],"not":[41],"included":[42],"in":[43,133],"-Maltese":[45],"-with":[46],"range":[48],"set":[51],"ups.We":[52],"conduct":[53],"evaluations":[54],"newly":[57,81],"pre-trained":[58],"on":[60,98,143,166],"three":[61],"morphosyntactic":[62],"tasks":[63],"-dependency":[64],"parsing,":[65],"part-of-speech":[66],"tagging,":[67],"and":[68,86,95,139,155,191],"named-entity":[69],"recognition":[70],"-and":[71],"one":[72],"semantic":[73],"classification":[74],"task":[75],"-sentiment":[76],"analysis.We":[77],"also":[78,119],"present":[79],"created":[82],"corpus":[83,126,172],"Maltese,":[85],"determine":[87],"size":[94],"domain":[96],"downstream":[100],"performance.Our":[101],"results":[102],"show":[103],"using":[105,115],"mixture":[107],"domains":[110],"often":[112],"superior":[113],"Wikipedia":[116],"text":[117],"only.We":[118],"find":[120],"fraction":[123],"enough":[128],"make":[130],"significant":[131],"leaps":[132],"performance":[134,165],"over":[135],"Wikipedia-trained":[136],"models.We":[137],"pre-train":[138],"compare":[140],"two":[141],"new":[145,171],"corpus:":[146],"BERT":[149,160],"model":[150],"trained":[151],"scratch":[153],"(BERTu),":[154],"further":[157],"pretrained":[158],"multilingual":[159],"(mBERTu).The":[161],"achieve":[163],"state-of-the-art":[164],"tasks,":[168],"despite":[169],"being":[173],"considerably":[174],"smaller":[175],"than":[176],"typically":[177],"used":[178],"corpora":[179],"high-resourced":[181],"languages.On":[182],"average,":[183],"BERTu":[184],"outperforms":[185],"or":[186],"performs":[187],"competitively":[188],"mBERTu,":[190],"largest":[193],"gains":[194],"are":[195],"observed":[196],"higher-level":[198],"tasks.":[199]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":1}],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2025-10-10T00:00:00"}