{"id":"https://openalex.org/W4416149152","doi":"https://doi.org/10.48550/arxiv.2502.19587","title":"NeoBERT: A Next-Generation BERT","display_name":"NeoBERT: A Next-Generation BERT","publication_year":2025,"publication_date":"2025-02-26","ids":{"openalex":"https://openalex.org/W4416149152","doi":"https://doi.org/10.48550/arxiv.2502.19587"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.19587","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.19587","pdf_url":"https://arxiv.org/pdf/2502.19587","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.19587","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120493958","display_name":"Lola Le Breton","orcid":"https://orcid.org/0009-0004-2119-0795","observed_orcids":["https://orcid.org/0009-0004-2119-0795"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Breton, Lola Le","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Fournier, Quentin","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fournier, Quentin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016869954","display_name":"Mariam El Mezouar","orcid":"https://orcid.org/0000-0002-3317-7051","observed_orcids":["https://orcid.org/0000-0002-3317-7051"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mezouar, Mariam El","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113290359","display_name":"John X. Morris","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Morris, John X.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5027720969","display_name":"Sarath Chandar","orcid":"https://orcid.org/0000-0002-9678-2830","observed_orcids":["https://orcid.org/0000-0002-9678-2830"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chandar, Sarath","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0017000000225380063,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0012000000569969416,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bidirectional-language-models","display_name":"bidirectional language models","score":0.8130000233650208},{"id":"https://openalex.org/keywords/pretraining","display_name":"pre-training","score":0.6150000095367432},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.781000018119812},{"id":"https://openalex.org/keywords/mteb-benchmark","display_name":"MTEB benchmark","score":0.843999981880188},{"id":"https://openalex.org/keywords/glue-benchmark","display_name":"GLUE benchmark","score":0.8009999990463257},{"id":"https://openalex.org/keywords/in-context-learning","display_name":"in-context learning","score":0.5920000076293945},{"id":"https://openalex.org/keywords/depth-to-width-ratio","display_name":"depth-to-width ratio","score":0.6510000228881836},{"id":"https://openalex.org/keywords/modernbert","display_name":"ModernBERT","score":0.9929999709129333}],"concepts":[{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.7434999942779541},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7235999703407288},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6723999977111816},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.617900013923645},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.519599974155426},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.46880000829696655},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.39879998564720154},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.3840999901294708},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.3797000050544739},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.35910001397132874},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.35260000824928284},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.3450999855995178},{"id":"https://openalex.org/C2776207758","wikidata":"https://www.wikidata.org/wiki/Q5303302","display_name":"Downstream (manufacturing)","level":2,"score":0.3034000098705292},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.29030001163482666},{"id":"https://openalex.org/C191172861","wikidata":"https://www.wikidata.org/wiki/Q7899321","display_name":"Upstream (networking)","level":2,"score":0.2816999852657318},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.25929999351501465},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.2551000118255615}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.19587","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.19587","pdf_url":"https://arxiv.org/pdf/2502.19587","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.19587","doi":"https://doi.org/10.48550/arxiv.2502.19587","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.19587","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.19587","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.19587","pdf_url":"https://arxiv.org/pdf/2502.19587","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4405626356","https://openalex.org/W7117696150","https://openalex.org/W7131632159","https://openalex.org/W3033187248","https://openalex.org/W4416620000","https://openalex.org/W6929338758","https://openalex.org/W7171693927","https://openalex.org/W7165623272","https://openalex.org/W4310561961","https://openalex.org/W3180942591"],"abstract_inverted_index":{"Recent":[0],"innovations":[1],"in":[2,71],"architecture,":[3,72],"pre-training,":[4],"and":[5,14,25,32,75,101,131,149,154,165,171],"fine-tuning":[6,135,153],"have":[7,34],"led":[8],"to":[9,168],"the":[10,37,62,121,142],"remarkable":[11],"in-context":[12],"learning":[13],"reasoning":[15],"abilities":[16],"of":[17,40,64,107,144],"large":[18],"auto-regressive":[19],"language":[20],"models":[21,66],"such":[22],"as":[23,87],"LLaMA":[24],"DeepSeek.":[26],"In":[27,137],"contrast,":[28],"encoders":[29],"like":[30],"BERT":[31,126],"RoBERTa":[33,128],"not":[35],"seen":[36],"same":[38],"level":[39],"progress":[41],"despite":[42],"being":[43],"foundational":[44],"for":[45,82,91,157],"many":[46],"downstream":[47],"NLP":[48],"applications.":[49],"To":[50],"bridge":[51],"this":[52],"gap,":[53],"we":[54,139],"introduce":[55],"NeoBERT,":[56],"a":[57,88,151],"next-generation":[58],"encoder":[59],"that":[60],"redefines":[61],"capabilities":[63],"bidirectional":[65],"by":[67],"integrating":[68],"state-of-the-art":[69,118],"advancements":[70],"modern":[73],"data,":[74,163],"optimized":[76],"pre-training":[77],"methodologies.":[78],"NeoBERT":[79],"is":[80],"designed":[81],"seamless":[83],"adoption:":[84],"it":[85,116],"serves":[86],"plug-and-play":[89],"replacement":[90],"existing":[92],"base":[93],"models,":[94],"relies":[95],"on":[96,120,147],"an":[97,103],"optimal":[98],"depth-to-width":[99],"ratio,":[100],"leverages":[102],"extended":[104],"context":[105],"length":[106],"4,096":[108],"tokens.":[109],"Despite":[110],"its":[111],"compact":[112],"250M":[113],"parameter":[114],"footprint,":[115],"achieves":[117],"results":[119],"massive":[122],"MTEB":[123],"benchmark,":[124],"outperforming":[125],"large,":[127,129],"NomicBERT,":[130],"ModernBERT":[132],"under":[133],"identical":[134],"conditions.":[136],"addition,":[138],"rigorously":[140],"evaluate":[141],"impact":[143],"each":[145],"modification":[146],"GLUE":[148],"design":[150],"uniform":[152],"evaluation":[155],"framework":[156],"MTEB.":[158],"We":[159],"release":[160],"all":[161],"code,":[162],"checkpoints,":[164],"training":[166],"scripts":[167],"accelerate":[169],"research":[170],"real-world":[172],"adoption.":[173]},"counts_by_year":[],"updated_date":"2026-09-29T14:09:16.559479","created_date":"2025-10-10T00:00:00"}