{"id":"https://openalex.org/W7135035775","doi":"https://doi.org/10.48550/arxiv.2603.09982","title":"AraModernBERT: Transtokenized Initialization and Long-Context Encoder Modeling for Arabic","display_name":"AraModernBERT: Transtokenized Initialization and Long-Context Encoder Modeling for Arabic","publication_year":2026,"publication_date":"2026-02-10","ids":{"openalex":"https://openalex.org/W7135035775","doi":"https://doi.org/10.48550/arxiv.2603.09982"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.09982","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09982","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.09982","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128867477","display_name":"Omar Elshehy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Elshehy, Omar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093933812","display_name":"Omer Nacar","orcid":"https://orcid.org/0000-0001-7493-9318"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nacar, Omer","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119752047","display_name":"Abdelbasset Djamai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Djamai, Abdelbasset","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128873741","display_name":"Muhammed Ragab","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ragab, Muhammed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088312471","display_name":"Khloud Al Jallad","orcid":"https://orcid.org/0000-0001-9474-9204"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jallad, Khloud Al","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5084111871","display_name":"Mona Abdelazim","orcid":"https://orcid.org/0000-0003-0069-9336"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abdelazim, Mona","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.4163999855518341,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.4163999855518341,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.27889999747276306,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12380","display_name":"Authorship Attribution and Profiling","score":0.03909999877214432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/discriminative-model","display_name":"Discriminative model","score":0.745199978351593},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.694599986076355},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.6470000147819519},{"id":"https://openalex.org/keywords/initialization","display_name":"Initialization","score":0.6186000108718872},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.49810001254081726},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.4542999863624573},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.436599999666214},{"id":"https://openalex.org/keywords/language-identification","display_name":"Language identification","score":0.383899986743927}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7803999781608582},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.745199978351593},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.694599986076355},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.6470000147819519},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6291999816894531},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.6186000108718872},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6061000227928162},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.49810001254081726},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.4542999863624573},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.436599999666214},{"id":"https://openalex.org/C129792486","wikidata":"https://www.wikidata.org/wiki/Q1050419","display_name":"Language identification","level":3,"score":0.383899986743927},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.3813000023365021},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.36640000343322754},{"id":"https://openalex.org/C176856949","wikidata":"https://www.wikidata.org/wiki/Q2001676","display_name":"Offensive","level":2,"score":0.33719998598098755},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.32989999651908875},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.3257000148296356},{"id":"https://openalex.org/C96455323","wikidata":"https://www.wikidata.org/wiki/Q13955","display_name":"Arabic","level":2,"score":0.28600001335144043},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.28450000286102295},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.27900001406669617},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.25690001249313354},{"id":"https://openalex.org/C94922259","wikidata":"https://www.wikidata.org/wiki/Q33215","display_name":"Constructed language","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.09982","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09982","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.09982","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09982","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.7686127424240112,"display_name":"Reduced inequalities"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Encoder-only":[0],"transformer":[1],"models":[2],"remain":[3],"widely":[4],"used":[5],"for":[6,56,126],"discriminative":[7,116],"NLP":[8],"tasks,":[9,100],"yet":[10],"recent":[11],"architectural":[12],"advances":[13],"have":[14],"largely":[15],"focused":[16],"on":[17,95],"English.":[18],"In":[19],"this":[20],"work,":[21],"we":[22],"present":[23],"AraModernBERT,":[24],"an":[25],"adaptation":[26],"of":[27,38],"the":[28,36],"ModernBERT":[29],"encoder":[30,129],"architecture":[31],"to":[32,47,69,115,131],"Arabic,":[33],"and":[34,42,79,108,117,133],"study":[35],"impact":[37],"transtokenized":[39],"embedding":[40],"initialization":[41],"native":[43],"long-context":[44,81],"modeling":[45,66,87],"up":[46],"8,192":[48],"tokens.":[49],"We":[50,72],"show":[51],"that":[52,75],"transtokenization":[53],"is":[54],"essential":[55],"Arabic":[57,96,132],"language":[58,65,86,98,104],"modeling,":[59,82],"yielding":[60],"dramatic":[61],"improvements":[62],"in":[63,137],"masked":[64],"performance":[67,88],"compared":[68],"non-transtokenized":[70],"initialization.":[71],"further":[73],"demonstrate":[74],"AraModernBERT":[76],"supports":[77],"stable":[78],"effective":[80],"achieving":[83],"improved":[84],"intrinsic":[85],"at":[89],"extended":[90],"sequence":[91,118],"lengths.":[92],"Downstream":[93],"evaluations":[94],"natural":[97],"understanding":[99],"including":[101],"inference,":[102],"offensive":[103],"detection,":[105],"question-question":[106],"similarity,":[107],"named":[109],"entity":[110],"recognition,":[111],"confirm":[112],"strong":[113],"transfer":[114],"labeling":[119],"settings.":[120],"Our":[121],"results":[122],"highlight":[123],"practical":[124],"considerations":[125],"adapting":[127],"modern":[128],"architectures":[130],"other":[134],"languages":[135],"written":[136],"Arabic-derived":[138],"scripts.":[139]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-13T00:00:00"}