{"id":"https://openalex.org/W4393399080","doi":"https://doi.org/10.48550/arxiv.2403.19887","title":"Jamba: A Hybrid Transformer-Mamba Language Model","display_name":"Jamba: A Hybrid Transformer-Mamba Language Model","publication_year":2024,"publication_date":"2024-03-28","ids":{"openalex":"https://openalex.org/W4393399080","doi":"https://doi.org/10.48550/arxiv.2403.19887"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2403.19887","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.19887","pdf_url":"https://arxiv.org/pdf/2403.19887","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2403.19887","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000625246","display_name":"Opher Lieber","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lieber, Opher","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041534013","display_name":"Barak Lenz","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lenz, Barak","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015802518","display_name":"Hofit Bata","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bata, Hofit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108947541","display_name":"Gal Cohen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cohen, Gal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024293285","display_name":"Jhonathan Osin","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Osin, Jhonathan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082565423","display_name":"Itay Dalmedigos","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dalmedigos, Itay","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094302512","display_name":"Erez Safahi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Safahi, Erez","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094302513","display_name":"Shaked Meirom","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Meirom, Shaked","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051184573","display_name":"Yonatan Belinkov","orcid":"https://orcid.org/0000-0002-6280-5045","observed_orcids":["https://orcid.org/0000-0002-6280-5045"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Belinkov, Yonatan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078263052","display_name":"Shai Shalev\u2010Shwartz","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shalev-Shwartz, Shai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059068945","display_name":"Omri Abend","orcid":"https://orcid.org/0000-0003-4311-3876","observed_orcids":["https://orcid.org/0000-0003-4311-3876"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abend, Omri","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094302505","display_name":"Raz Alon","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alon, Raz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120304517","display_name":"Tomer Asida","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Asida, Tomer","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113157690","display_name":"Amir Bergman","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bergman, Amir","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094302507","display_name":"Roman Glozman","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Glozman, Roman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094302508","display_name":"Michael Gokhman","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gokhman, Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094302509","display_name":"Avashalom Manevich","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Manevich, Avashalom","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005163180","display_name":"Nir Ratner","orcid":"https://orcid.org/0000-0003-2305-897X","observed_orcids":["https://orcid.org/0000-0003-2305-897X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ratner, Nir","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060961149","display_name":"Noam Rozen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rozen, Noam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046521761","display_name":"Erez Schwartz","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shwartz, Erez","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094302511","display_name":"Mor Zusman","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zusman, Mor","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5028913764","display_name":"Yoav Shoham","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shoham, Yoav","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":70,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8187999725341797,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8187999725341797,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.13609999418258667,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.012199999764561653,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mixture-of-experts","display_name":"mixture of experts","score":0.8100000023841858},{"id":"https://openalex.org/keywords/mamba","display_name":"Mamba","score":0.7239999771118164},{"id":"https://openalex.org/keywords/long-context-language-models","display_name":"long-context language models","score":0.6740000247955322},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6800000071525574},{"id":"https://openalex.org/keywords/memory-efficient-inference","display_name":"memory-efficient inference","score":0.5180000066757202},{"id":"https://openalex.org/keywords/ablation-study","display_name":"ablation study","score":0.44600000977516174}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6151467561721802},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3730102777481079},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1717292070388794},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.16918912529945374},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.08842357993125916}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2403.19887","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.19887","pdf_url":"https://arxiv.org/pdf/2403.19887","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2403.19887","doi":"https://doi.org/10.48550/arxiv.2403.19887","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2403.19887","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2403.19887","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.19887","pdf_url":"https://arxiv.org/pdf/2403.19887","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393399080.pdf","grobid_xml":"https://content.openalex.org/works/W4393399080.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4405664040","https://openalex.org/W4399657245","https://openalex.org/W4417054624","https://openalex.org/W7163879968","https://openalex.org/W4403346143","https://openalex.org/W4416445253","https://openalex.org/W4399115929","https://openalex.org/W4414972737","https://openalex.org/W7222585852","https://openalex.org/W4415797412"],"abstract_inverted_index":{"We":[0,125,156,191],"present":[1],"Jamba,":[2],"a":[3,11,71,77,203],"new":[4],"base":[5],"large":[6,83,153],"language":[7,106],"model":[8,32,44,73,107,114],"based":[9],"on":[10,104],"novel":[12,189],"hybrid":[13],"Transformer-Mamba":[14],"mixture-of-experts":[15],"(MoE)":[16],"architecture.":[17,190],"Specifically,":[18],"Jamba":[19,85,171,199],"interleaves":[20],"blocks":[21],"of":[22,30,39,148,162,170,187,195,198],"Transformer":[23,135],"and":[24,57,89,97,109,136,139,144,168,174],"Mamba":[25,137],"layers,":[26,138],"enjoying":[27],"the":[28,61,99,113,166,193],"benefits":[29],"both":[31],"families.":[33],"MoE":[34],"is":[35],"added":[36],"in":[37,76,152],"some":[38,147],"these":[40,163],"layers":[41],"to":[42,94,120,133,141,176,183],"increase":[43],"capacity":[45],"while":[46],"keeping":[47],"active":[48],"parameter":[49],"usage":[50],"manageable.":[51],"This":[52],"flexible":[53],"architecture":[54],"allows":[55],"resource-":[56],"objective-specific":[58],"configurations.":[59],"In":[60],"particular":[62],"configuration":[63],"we":[64,67],"have":[65,172],"implemented,":[66],"end":[68],"up":[69,119],"with":[70],"powerful":[72],"that":[74,146],"fits":[75],"single":[78],"80GB":[79],"GPU.":[80],"Built":[81],"at":[82,98],"scale,":[84],"provides":[86],"high":[87],"throughput":[88],"small":[90],"memory":[91],"footprint":[92],"compared":[93],"vanilla":[95],"Transformers,":[96],"same":[100],"time":[101],"state-of-the-art":[102],"performance":[103],"standard":[105],"benchmarks":[108],"long-context":[110],"evaluations.":[111],"Remarkably,":[112],"presents":[115],"strong":[116],"results":[117],"for":[118],"256K":[121],"tokens":[122],"context":[123],"length.":[124],"study":[126],"various":[127,180],"architectural":[128],"decisions,":[129],"such":[130],"as":[131],"how":[132,140],"combine":[134],"mix":[142],"experts,":[143],"show":[145],"them":[149],"are":[150],"crucial":[151],"scale":[154],"modeling.":[155],"also":[157],"describe":[158],"several":[159],"interesting":[160],"properties":[161],"architectures":[164],"which":[165],"training":[167],"evaluation":[169],"revealed,":[173],"plan":[175],"release":[177],"checkpoints":[178],"from":[179],"ablation":[181],"runs,":[182],"encourage":[184],"further":[185],"exploration":[186],"this":[188],"make":[192],"weights":[194],"our":[196],"implementation":[197],"publicly":[200],"available":[201],"under":[202],"permissive":[204],"license.":[205]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":44},{"year":2024,"cited_by_count":23}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2024-04-02T00:00:00"}