{"id":"https://openalex.org/W4402427536","doi":"https://doi.org/10.48550/arxiv.2408.07180","title":"Unlocking Efficiency: Adaptive Masking for Gene Transformer Models","display_name":"Unlocking Efficiency: Adaptive Masking for Gene Transformer Models","publication_year":2024,"publication_date":"2024-08-13","ids":{"openalex":"https://openalex.org/W4402427536","doi":"https://doi.org/10.48550/arxiv.2408.07180"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2408.07180","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.07180","pdf_url":"https://arxiv.org/pdf/2408.07180","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2408.07180","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010730449","display_name":"Soumyadeep Roy","orcid":"https://orcid.org/0000-0001-7269-2163"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Roy, Soumyadeep","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036571304","display_name":"Shamik Sural","orcid":"https://orcid.org/0000-0002-4315-7329"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sural, Shamik","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5073812421","display_name":"Niloy Ganguly","orcid":"https://orcid.org/0000-0002-3967-186X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ganguly, Niloy","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5010730449"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11975","display_name":"Evolutionary Algorithms and Applications","score":0.8432999849319458,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11975","display_name":"Evolutionary Algorithms and Applications","score":0.8432999849319458,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.7649999856948853,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10621","display_name":"Gene Regulatory Network Analysis","score":0.7340999841690063,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/masking","display_name":"Masking (illustration)","score":0.6021397113800049},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5906952619552612},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.48960238695144653},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.22898384928703308},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.14448282122612},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.08285602927207947},{"id":"https://openalex.org/keywords/art","display_name":"Art","score":0.05793103575706482}],"concepts":[{"id":"https://openalex.org/C2777402240","wikidata":"https://www.wikidata.org/wiki/Q6783436","display_name":"Masking (illustration)","level":2,"score":0.6021397113800049},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5906952619552612},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48960238695144653},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.22898384928703308},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.14448282122612},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.08285602927207947},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.05793103575706482},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2408.07180","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.07180","pdf_url":"https://arxiv.org/pdf/2408.07180","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2408.07180","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2408.07180","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2408.07180","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.07180","pdf_url":"https://arxiv.org/pdf/2408.07180","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4402427536.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"Gene":[0,132],"transformer":[1],"models":[2,182,226],"such":[3,45],"as":[4,46,113],"Nucleotide":[5,184],"Transformer,":[6],"DNABert,":[7],"and":[8,82,163,197,227],"LOGO":[9,205],"are":[10,73],"trained":[11,187],"to":[12,50,66,75,121,142],"learn":[13],"optimal":[14],"gene":[15,114,150],"sequence":[16,151],"representations":[17],"by":[18,105],"using":[19,91,106],"the":[20,28,34,58,98,215,225],"Masked":[21],"Language":[22],"Modeling":[23],"(MLM)":[24],"training":[25,71],"objective":[26],"over":[27],"complete":[29],"Human":[30],"Reference":[31],"Genome.":[32],"However,":[33],"typical":[35],"tokenization":[36],"methods":[37],"employ":[38],"a":[39,107],"basic":[40],"sliding":[41],"window":[42],"of":[43,61,100,125,172,214,219],"tokens,":[44],"k-mers,":[47],"that":[48,178,203],"fail":[49],"utilize":[51],"gene-centric":[52],"semantics.":[53],"This":[54],"could":[55],"result":[56],"in":[57,79,158,194],"(trivial)":[59],"masking":[60,93,144],"easily":[62],"predictable":[63],"sequences,":[64],"leading":[65],"inefficient":[67],"MLM":[68],"training.":[69],"Time-variant":[70],"strategies":[72],"known":[74],"improve":[76],"pretraining":[77],"efficiency":[78],"both":[80,159],"language":[81],"vision":[83],"tasks.":[84,153],"In":[85],"this":[86],"work,":[87],"we":[88,95],"focus":[89],"on":[90,148],"curriculum":[92],"where":[94],"systematically":[96],"increase":[97],"difficulty":[99,111],"masked":[101],"token":[102],"prediction":[103],"task":[104],"Pointwise":[108],"Mutual":[109],"Information-based":[110],"criterion,":[112],"sequences":[115],"lack":[116],"well-defined":[117],"semantic":[118],"units":[119],"similar":[120,192],"words":[122],"or":[123],"sentences":[124],"NLP":[126],"domain.":[127],"Our":[128,175],"proposed":[129],"Curriculum":[130],"Masking-based":[131],"Masking":[133],"Strategy":[134],"(CM-GEMS)":[135],"demonstrates":[136],"superior":[137],"representation":[138],"learning":[139],"capabilities":[140],"compared":[141],"baseline":[143],"approaches":[145],"when":[146],"evaluated":[147],"downstream":[149],"classification":[152],"We":[154,200,222],"perform":[155],"extensive":[156],"evaluation":[157],"few-shot":[160],"(five":[161],"datasets)":[162],"full":[164],"dataset":[165],"settings":[166],"(Genomic":[167],"Understanding":[168],"Evaluation":[169],"benchmark":[170],"consisting":[171],"27":[173],"tasks).":[174],"findings":[176],"reveal":[177],"CM-GEMS":[179],"outperforms":[180],"state-of-the-art":[181,216],"(DNABert-2,":[183],"transformer,":[185],"DNABert)":[186],"at":[188,231],"120K":[189,220],"steps,":[190],"achieving":[191],"results":[193],"just":[195],"10K":[196],"1K":[198],"steps.":[199,221],"also":[201],"demonstrate":[202],"Curriculum-Learned":[204],"(a":[206],"2-layer":[207],"DNABert-like":[208],"model)":[209],"can":[210],"achieve":[211],"nearly":[212],"90%":[213],"model":[217],"performance":[218],"will":[223],"make":[224],"codes":[228],"publicly":[229],"available":[230],"https://github.com/roysoumya/curriculum-GeneMask.":[232]},"counts_by_year":[],"updated_date":"2026-03-11T14:59:36.786465","created_date":"2025-10-10T00:00:00"}