{"id":"https://openalex.org/W4404449573","doi":"https://doi.org/10.48550/arxiv.2411.08909","title":"Long-context Protein Language Modeling Using Bidirectional Mamba with Shared Projection Layers","display_name":"Long-context Protein Language Modeling Using Bidirectional Mamba with Shared Projection Layers","publication_year":2024,"publication_date":"2024-10-29","ids":{"openalex":"https://openalex.org/W4404449573","doi":"https://doi.org/10.48550/arxiv.2411.08909"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2411.08909","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.08909","pdf_url":"https://arxiv.org/pdf/2411.08909","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2411.08909","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054001148","display_name":"Yingheng Wang","orcid":"https://orcid.org/0000-0002-4828-4757"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yingheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100784336","display_name":"Zichen Wang","orcid":"https://orcid.org/0000-0002-1415-1286"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zichen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077723460","display_name":"Gil Sadeh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sadeh, Gil","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036782268","display_name":"Luca Zancato","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zancato, Luca","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065386783","display_name":"Alessandro Achille","orcid":"https://orcid.org/0000-0002-8163-8326"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Achille, Alessandro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082384108","display_name":"George Karypis","orcid":"https://orcid.org/0000-0003-2753-1437"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karypis, George","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5006581225","display_name":"Huzefa Rangwala","orcid":"https://orcid.org/0000-0003-0435-0035"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rangwala, Huzefa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12254","display_name":"Machine Learning in Bioinformatics","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T12254","display_name":"Machine Learning in Bioinformatics","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T13937","display_name":"Genetics, Bioinformatics, and Biomedical Research","score":0.9210000038146973,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5700508952140808},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.45614099502563477},{"id":"https://openalex.org/keywords/history","display_name":"History","score":0.08817324042320251}],"concepts":[{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5700508952140808},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.45614099502563477},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.08817324042320251},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2411.08909","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.08909","pdf_url":"https://arxiv.org/pdf/2411.08909","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2411.08909","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.08909","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2411.08909","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.08909","pdf_url":"https://arxiv.org/pdf/2411.08909","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404449573.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Self-supervised":[0],"training":[1],"of":[2,135,176,194],"language":[3,116],"models":[4],"(LMs)":[5],"has":[6],"seen":[7],"great":[8],"success":[9],"for":[10,18,56,131],"protein":[11,23,40,48,92,106,155,183,212],"sequences":[12],"in":[13,78,209,220],"learning":[14,210],"meaningful":[15],"representations":[16,107,213],"and":[17,47,66,76,147,151,166,185,214],"generative":[19],"drug":[20],"design.":[21],"Most":[22],"LMs":[24,41],"are":[25],"based":[26,88],"on":[27,32,89,154,182],"the":[28,57,109,174,192,196],"Transformer":[29],"architecture":[30,203],"trained":[31,163,172],"individual":[33],"proteins":[34,46,69],"with":[35,72,164,199],"short":[36],"context":[37,175,197],"lengths.":[38],"Such":[39],"cannot":[42],"extrapolate":[43],"to":[44,54,102,149,159],"longer":[45],"complexes":[49],"well.":[50],"They":[51],"also":[52,119],"fail":[53],"account":[55],"underlying":[58],"biological":[59,80,221],"mechanisms":[60],"carried":[61],"out":[62],"by":[63],"biomolecular":[64],"interactions":[65],"dynamics":[67],"i.e.,":[68],"often":[70],"interact":[71],"other":[73],"proteins,":[74],"molecules,":[75],"pathways":[77],"complex":[79],"systems.":[81],"In":[82],"this":[83],"work,":[84],"we":[85],"propose":[86],"LC-PLM":[87,137],"an":[90],"alternative":[91],"LM":[93,202],"architecture,":[94],"BiMamba-S,":[95],"built":[96],"upon":[97],"selective":[98],"structured":[99,205],"state-space":[100],"models,":[101],"learn":[103],"high-quality":[104],"universal":[105,211],"at":[108],"amino":[110],"acid":[111],"token":[112],"level":[113],"using":[114],"masked":[115],"modeling.":[117],"We":[118],"introduce":[120],"its":[121],"graph-contextual":[122],"variant,":[123],"LC-PLM,":[124],"which":[125],"contextualizes":[126],"protein-protein":[127],"interaction":[128,217],"(PPI)":[129],"graphs":[130,178],"a":[132],"second":[133],"stage":[134],"training.":[136],"demonstrates":[138,191],"favorable":[139],"neural":[140],"scaling":[141],"laws,":[142],"better":[143],"length":[144],"extrapolation":[145],"capability,":[146],"up":[148],"30%":[150],"16%":[152],"improvements":[153],"downstream":[156],"tasks":[157],"compared":[158],"Transformer-based":[160],"ESM-2":[161],"when":[162],"100B":[165],"1T":[167],"tokens,":[168],"respectively.":[169],"LC-PLM-G":[170],"further":[171],"within":[173],"PPI":[177],"shows":[179],"promising":[180],"results":[181],"structure":[184],"function":[186],"prediction":[187],"tasks.":[188],"Our":[189],"study":[190],"benefit":[193],"increasing":[195],"size":[198],"computationally":[200],"efficient":[201],"(e.g.,":[204],"state":[206],"space":[207],"models)":[208],"incorporating":[215],"molecular":[216],"contexts":[218],"contained":[219],"graphs.":[222]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2024-11-17T00:00:00"}