{"id":"https://openalex.org/W4417091809","doi":"https://doi.org/10.48550/arxiv.2504.13181","title":"Perception Encoder: The best visual embeddings are not at the output of the network","display_name":"Perception Encoder: The best visual embeddings are not at the output of the network","publication_year":2025,"publication_date":"2025-04-17","ids":{"openalex":"https://openalex.org/W4417091809","doi":"https://doi.org/10.48550/arxiv.2504.13181"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2504.13181","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.13181","pdf_url":"https://arxiv.org/pdf/2504.13181","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2504.13181","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063326998","display_name":"Daniel Bolya","orcid":"https://orcid.org/0000-0003-0223-3599"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Bolya, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063149046","display_name":"Po-Yao Huang","orcid":"https://orcid.org/0000-0002-3319-5145"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Po-Yao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002942097","display_name":"Peize Sun","orcid":"https://orcid.org/0000-0003-0636-2224"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Peize","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066881642","display_name":"Jang Hyun Cho","orcid":"https://orcid.org/0000-0002-3654-7505"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cho, Jang Hyun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061096363","display_name":"Andrea Madotto","orcid":"https://orcid.org/0000-0002-8672-715X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Madotto, Andrea","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100756973","display_name":"Chen Wei","orcid":"https://orcid.org/0000-0003-0486-3084"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101821970","display_name":"Tengyu Ma","orcid":"https://orcid.org/0000-0003-3916-5040"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Tengyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031575313","display_name":"Jiale Zhi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhi, Jiale","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053765366","display_name":"Jathushan Rajasegaran","orcid":"https://orcid.org/0000-0003-1081-4254"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rajasegaran, Jathushan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064948724","display_name":"Hanoona Rasheed","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rasheed, Hanoona","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101517372","display_name":"Junke Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Junke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010633303","display_name":"Marco Aur\u00e9lio Alvarenga Monteiro","orcid":"https://orcid.org/0000-0002-4426-1638"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Monteiro, Marco","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100385219","display_name":"Xu Hu","orcid":"https://orcid.org/0000-0002-6345-2448"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Hu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100645333","display_name":"Shiyu Dong","orcid":"https://orcid.org/0000-0001-9335-9132"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Shiyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045356290","display_name":"Nikhila Ravi","orcid":"https://orcid.org/0000-0003-0097-5222"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ravi, Nikhila","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047759179","display_name":"Daniel Li","orcid":"https://orcid.org/0000-0002-2358-8180"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057866698","display_name":"Piotr Doll\u00e1r","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Doll\u00e1r, Piotr","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5036069974","display_name":"Christoph Feichtenhofer","orcid":"https://orcid.org/0000-0001-9756-7238"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feichtenhofer, Christoph","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":18,"corresponding_author_ids":["https://openalex.org/A5063326998"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.7186999917030334,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.7186999917030334,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.12269999831914902,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.04259999841451645,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.7631000280380249},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.6552000045776367},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.5936999917030334},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.483599990606308},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.3774000108242035},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.37549999356269836},{"id":"https://openalex.org/keywords/image-processing","display_name":"Image processing","score":0.36809998750686646},{"id":"https://openalex.org/keywords/visual-perception","display_name":"Visual perception","score":0.3440999984741211}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7754999995231628},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.7631000280380249},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6718999743461609},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6552000045776367},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.5936999917030334},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.483599990606308},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4641999900341034},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.3774000108242035},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.37549999356269836},{"id":"https://openalex.org/C9417928","wikidata":"https://www.wikidata.org/wiki/Q1070689","display_name":"Image processing","level":3,"score":0.36809998750686646},{"id":"https://openalex.org/C178253425","wikidata":"https://www.wikidata.org/wiki/Q162668","display_name":"Visual perception","level":3,"score":0.3440999984741211},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.3237999975681305},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3163999915122986},{"id":"https://openalex.org/C2779757391","wikidata":"https://www.wikidata.org/wiki/Q6002292","display_name":"Image translation","level":3,"score":0.29820001125335693},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.29600000381469727},{"id":"https://openalex.org/C75294576","wikidata":"https://www.wikidata.org/wiki/Q5165192","display_name":"Contextual image classification","level":3,"score":0.2930999994277954},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.2912999987602234},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.28519999980926514},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.28049999475479126},{"id":"https://openalex.org/C160086991","wikidata":"https://www.wikidata.org/wiki/Q5939193","display_name":"Human visual system model","level":3,"score":0.2768999934196472},{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.2540999948978424},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.25209999084472656}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2504.13181","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.13181","pdf_url":"https://arxiv.org/pdf/2504.13181","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2504.13181","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2504.13181","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2504.13181","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.13181","pdf_url":"https://arxiv.org/pdf/2504.13181","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4417091809.pdf","grobid_xml":"https://content.openalex.org/works/W4417091809.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0],"introduce":[1,98],"Perception":[2],"Encoder":[3],"(PE),":[4],"a":[5,25,124,181],"state-of-the-art":[6,184],"vision":[7,20],"encoder":[8],"for":[9,71,104,111],"image":[10,48,132],"and":[11,51,108,133,136,145,154,162,169,177,198,203],"video":[12,56,134,149,155],"understanding":[13],"trained":[14],"via":[15],"simple":[16],"vision-language":[17,63],"learning.":[18],"Traditionally,":[19],"encoders":[21],"have":[22],"relied":[23],"on":[24,123],"variety":[26,126],"of":[27,73,90,118,127,185,201],"pretraining":[28,49],"objectives,":[29],"each":[30],"tailored":[31],"to":[32],"specific":[33],"downstream":[34,75],"tasks":[35,172],"such":[36,173],"as":[37,174],"classification,":[38],"captioning,":[39],"or":[40],"localization.":[41],"Surprisingly,":[42],"after":[43],"scaling":[44],"our":[45,54,115,195],"carefully":[46],"tuned":[47],"recipe":[50],"refining":[52],"with":[53,165],"robust":[55],"data":[57],"engine,":[58],"we":[59,97,193],"find":[60],"that":[61],"contrastive":[62],"training":[64],"alone":[65],"can":[66],"produce":[67],"strong,":[68],"general":[69],"embeddings":[70,83],"all":[72],"these":[74,82],"tasks.":[76],"There":[77],"is":[78],"only":[79],"one":[80],"caveat:":[81],"are":[84],"hidden":[85],"within":[86],"the":[87,91],"intermediate":[88],"layers":[89],"network.":[92],"To":[93,189],"draw":[94],"them":[95],"out,":[96],"two":[99],"alignment":[100,103,110],"methods:":[101],"language":[102,106],"multimodal":[105],"modeling,":[107],"spatial":[109,171],"dense":[112],"prediction.":[113],"Together,":[114],"PE":[116],"family":[117],"models":[119],"achieves":[120],"best-in-class":[121],"results":[122],"wide":[125],"tasks,":[128],"including":[129],"(1)":[130],"zero-shot":[131,142,147],"classification":[135],"retrieval,":[137],"simultaneously":[138],"obtaining":[139],"86.6":[140],"average":[141],"ImageNet":[143],"robustness":[144],"76.9":[146],"Kinetics-400":[148],"classification;":[150],"(2)":[151],"document,":[152],"image,":[153],"Q&A,":[156],"enabling":[157],"94.6":[158],"DocVQA,":[159],"80.9":[160],"InfographicVQA,":[161],"82.7":[163],"PerceptionTest":[164],"an":[166],"8B":[167],"LLM;":[168],"(3)":[170],"detection,":[175],"tracking,":[176],"depth":[178],"estimation,":[179],"setting":[180],"new":[182],"COCO":[183],"66.0":[186],"box":[187],"mAP.":[188],"foster":[190],"further":[191],"research,":[192],"release":[194],"models,":[196],"code,":[197],"novel":[199],"dataset":[200],"synthetically":[202],"human-annotated":[204],"videos:":[205],"https://github.com/facebookresearch/perception_models":[206]},"counts_by_year":[],"updated_date":"2026-05-05T08:41:31.759640","created_date":"2025-10-10T00:00:00"}