{"id":"https://openalex.org/W4283074957","doi":"https://doi.org/10.48550/arxiv.2206.08356","title":"OmniMAE: Single Model Masked Pretraining on Images and Videos","display_name":"OmniMAE: Single Model Masked Pretraining on Images and Videos","publication_year":2022,"publication_date":"2022-06-16","ids":{"openalex":"https://openalex.org/W4283074957","doi":"https://doi.org/10.48550/arxiv.2206.08356"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2206.08356","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2206.08356","pdf_url":"https://arxiv.org/pdf/2206.08356","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2206.08356","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006312307","display_name":"Rohit Girdhar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Girdhar, Rohit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083505661","display_name":"Alaaeldin El-Nouby","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"El-Nouby, Alaaeldin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060023106","display_name":"Mannat Singh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Singh, Mannat","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078659702","display_name":"Kalyan Vasudev Alwala","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alwala, Kalyan Vasudev","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107859338","display_name":"Armand Joulin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Joulin, Armand","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5000623592","display_name":"Ishan Misra","orcid":"https://orcid.org/0000-0001-7708-7261"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Misra, Ishan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11019","display_name":"Image Enhancement Techniques","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11019","display_name":"Image Enhancement Techniques","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8778968453407288},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.6942133903503418},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.6494772434234619},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6042011976242065},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5779094696044922},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5590932965278625},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.5059532523155212},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.42505255341529846},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.41272062063217163}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8778968453407288},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.6942133903503418},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.6494772434234619},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6042011976242065},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5779094696044922},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5590932965278625},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.5059532523155212},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.42505255341529846},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.41272062063217163},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2206.08356","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2206.08356","pdf_url":"https://arxiv.org/pdf/2206.08356","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2206.08356","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2206.08356","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2206.08356","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2206.08356","pdf_url":"https://arxiv.org/pdf/2206.08356","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/11","display_name":"Sustainable cities and communities","score":0.800000011920929}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4283074957.pdf","grobid_xml":"https://content.openalex.org/works/W4283074957.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W73545470","https://openalex.org/W4224266612","https://openalex.org/W2383394264","https://openalex.org/W4320153225","https://openalex.org/W4293261942","https://openalex.org/W3125968744","https://openalex.org/W2167701463","https://openalex.org/W2110287964","https://openalex.org/W4307407935","https://openalex.org/W649759291"],"abstract_inverted_index":{"Transformer-based":[0],"architectures":[1,48],"have":[2],"become":[3],"competitive":[4],"across":[5],"a":[6,25,33,75,111,168],"variety":[7],"of":[8,124,129,137],"visual":[9,39,92],"domains,":[10],"most":[11],"notably":[12],"images":[13,80],"and":[14,81,106,127,158],"videos.":[15],"While":[16],"prior":[17],"work":[18],"studies":[19],"these":[20],"modalities":[21],"in":[22],"isolation,":[23],"having":[24],"common":[26],"architecture":[27],"suggests":[28],"that":[29,67,94,145],"one":[30],"can":[31,70,118,150],"train":[32,74],"single":[34,59,89,147],"unified":[35,44],"model":[36,90,117,139,149],"for":[37,50],"multiple":[38],"modalities.":[40],"Prior":[41],"attempts":[42],"at":[43],"modeling":[45],"typically":[46],"use":[47],"tailored":[49],"vision":[51],"tasks,":[52],"or":[53,98],"obtain":[54],"worse":[55],"performance":[56],"compared":[57],"to":[58,73,97,153],"modality":[60],"models.":[61],"In":[62,141],"this":[63,116],"work,":[64],"we":[65,143],"show":[66,144],"masked":[68],"autoencoding":[69],"be":[71,119,151],"used":[72],"simple":[76],"Vision":[77],"Transformer":[78],"on":[79,103,156,160],"videos,":[82],"without":[83],"requiring":[84],"any":[85],"labeled":[86],"data.":[87],"This":[88],"learns":[91],"representations":[93,102],"are":[95],"comparable":[96],"better":[99],"than":[100],"single-modality":[101],"both":[104],"image":[105,126],"video":[107,131,165],"benchmarks,":[108],"while":[109],"using":[110],"much":[112],"simpler":[113],"architecture.":[114],"Furthermore,":[115],"learned":[120],"by":[121],"dropping":[122],"90%":[123],"the":[125,130,161],"95%":[128],"patches,":[132],"enabling":[133],"extremely":[134],"fast":[135],"training":[136],"huge":[138],"architectures.":[140],"particular,":[142],"our":[146],"ViT-Huge":[148],"finetuned":[152],"achieve":[154],"86.6%":[155],"ImageNet":[157],"75.5%":[159],"challenging":[162],"Something":[163],"Something-v2":[164],"benchmark,":[166],"setting":[167],"new":[169],"state-of-the-art.":[170]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}