{"id":"https://openalex.org/W4385963671","doi":"https://doi.org/10.48550/arxiv.2308.01313","title":"PerceptionCLIP: Visual Classification by Inferring and Conditioning on Contexts","display_name":"PerceptionCLIP: Visual Classification by Inferring and Conditioning on Contexts","publication_year":2023,"publication_date":"2023-08-02","ids":{"openalex":"https://openalex.org/W4385963671","doi":"https://doi.org/10.48550/arxiv.2308.01313"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2308.01313","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.01313","pdf_url":"https://arxiv.org/pdf/2308.01313","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2308.01313","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101402110","display_name":"Bang An","orcid":"https://orcid.org/0000-0002-2268-6010","observed_orcids":["https://orcid.org/0000-0002-2268-6010"]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"An, Bang","raw_affiliation_strings":["University of Maryland , College Park"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland , College Park","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065758953","display_name":"Sicheng Zhu","orcid":"https://orcid.org/0000-0002-8608-4262","observed_orcids":["https://orcid.org/0000-0002-8608-4262"]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhu, Sicheng","raw_affiliation_strings":["University of Maryland , College Park"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland , College Park","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073224776","display_name":"Michael-Andrei Panaitescu-Liess","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Panaitescu-Liess, Michael-Andrei","raw_affiliation_strings":["University of Maryland , College Park"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland , College Park","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077132380","display_name":"Chaithanya Kumar Mummadi","orcid":"https://orcid.org/0000-0002-1173-2720","observed_orcids":["https://orcid.org/0000-0002-1173-2720"]},"institutions":[{"id":"https://openalex.org/I4210120115","display_name":"Robert Bosch (United States)","ror":"https://ror.org/02venad53","country_code":"US","type":"company","lineage":["https://openalex.org/I4210120115","https://openalex.org/I889804353"]},{"id":"https://openalex.org/I4210151956","display_name":"Robert Bosch (India)","ror":"https://ror.org/04my8ty22","country_code":"IN","type":"company","lineage":["https://openalex.org/I4210151956","https://openalex.org/I889804353"]}],"countries":["IN","US"],"is_corresponding":false,"raw_author_name":"Mummadi, Chaithanya Kumar","raw_affiliation_strings":["Bosch Center for Artificial Intelligence"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bosch Center for Artificial Intelligence","institution_ids":["https://openalex.org/I4210120115","https://openalex.org/I4210151956"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037162168","display_name":"Furong Huang","orcid":"https://orcid.org/0000-0002-7393-1047","observed_orcids":["https://orcid.org/0000-0002-7393-1047"]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Huang, Furong","raw_affiliation_strings":["University of Maryland , College Park"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland , College Park","institution_ids":["https://openalex.org/I66946132"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.8062000274658203,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.8062000274658203,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.16249999403953552,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.024000000208616257,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/zero-shot-image-classification","display_name":"zero-shot image classification","score":0.9380000233650208},{"id":"https://openalex.org/keywords/clip","display_name":"CLIP","score":0.9409999847412109},{"id":"https://openalex.org/keywords/foreground-background-separation","display_name":"foreground-background separation","score":0.6830000281333923},{"id":"https://openalex.org/keywords/spurious-features","display_name":"spurious features","score":0.7070000171661377},{"id":"https://openalex.org/keywords/training-free-classification","display_name":"training-free classification","score":0.7129999995231628},{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.6819999814033508},{"id":"https://openalex.org/keywords/visual-perception","display_name":"visual perception","score":0.6499999761581421}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7896816730499268},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6893315315246582},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.5714048743247986},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.552418053150177},{"id":"https://openalex.org/C75294576","wikidata":"https://www.wikidata.org/wiki/Q5165192","display_name":"Contextual image classification","level":3,"score":0.5241004228591919},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.4885641038417816},{"id":"https://openalex.org/C2776378700","wikidata":"https://www.wikidata.org/wiki/Q3030775","display_name":"Distraction","level":2,"score":0.46943190693855286},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.44594988226890564},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.43079453706741333},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.42827799916267395},{"id":"https://openalex.org/C64876066","wikidata":"https://www.wikidata.org/wiki/Q5141226","display_name":"Cognitive neuroscience of visual object recognition","level":3,"score":0.4175173044204712},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.4106515049934387},{"id":"https://openalex.org/C2776151529","wikidata":"https://www.wikidata.org/wiki/Q3045304","display_name":"Object detection","level":3,"score":0.4104061722755432},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39642593264579773},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.38880836963653564},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3823195993900299},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.3561249375343323},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10232442617416382},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2308.01313","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.01313","pdf_url":"https://arxiv.org/pdf/2308.01313","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2308.01313","doi":"https://doi.org/10.48550/arxiv.2308.01313","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2308.01313","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2308.01313","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.01313","pdf_url":"https://arxiv.org/pdf/2308.01313","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"display_name":"Quality education","score":0.5199999809265137,"id":"https://metadata.un.org/sdg/4"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G1523888516","display_name":null,"funder_award_id":"FA9550-","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G4720003262","display_name":null,"funder_award_id":"N00014-22","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G4805816520","display_name":null,"funder_award_id":"FA9550","funder_id":"https://openalex.org/F4320306078","funder_display_name":"U.S. Department of Defense"},{"id":"https://openalex.org/G5702281522","display_name":"FAI: Toward Fair Decision Making and Resource Allocation with Application to AI-Assisted Graduate Admission and Degree Completion","funder_award_id":"2147276","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6098521345","display_name":null,"funder_award_id":"FA9550","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G7437616762","display_name":null,"funder_award_id":"HR00112020007","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G8259519366","display_name":"TRUSTWORTHY REAL-TIME COMMUNICATIVE MULTI-AGENT REINFORCEMENT LEARNING FOR DECISION-MAKING IN UNSTABLE & ADVERSARIAL SYSTEMS","funder_award_id":"N000142212335","funder_id":"https://openalex.org/F4320306078","funder_display_name":"U.S. Department of Defense"},{"id":"https://openalex.org/G8330669556","display_name":"AUTONOMOUS DECISION-MAKING SYSTEMS UNDER (ADVERSARIAL) DISTRIBUTION SHIFTS: LEARNING TO ADAPT, GENERALIZE AND ROBUSTIFY","funder_award_id":"FA95502310048","funder_id":"https://openalex.org/F4320306078","funder_display_name":"U.S. Department of Defense"},{"id":"https://openalex.org/G8689372435","display_name":"THE PURPOSE OF THIS AGREEMENT IS TO FUND RESEARCH SUPPORTING THE DEFENSE ADVANCED RESEARCH PROJECTS AGENCY DARPA GUARANTEEING AI ROBUSTNESS AGAINST DECEPTION GARD PROGRAM. THE TERM OF THE BASE PERIOD FOR THIS AGREEMENT COMMENCES ON THE EFFECTIVE DATE OF THE AWARD AND CONTINUES FOR TWELVE 12 MONTHS AFTER AWARD.","funder_award_id":"HR00112020007","funder_id":"https://openalex.org/F4320306078","funder_display_name":"U.S. Department of Defense"},{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306078","display_name":"U.S. Department of Defense","ror":"https://ror.org/0447fe631"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320332815","display_name":"Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"},{"id":"https://openalex.org/F4320338279","display_name":"Air Force Office of Scientific Research","ror":"https://ror.org/011e9bt93"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385963671.pdf","grobid_xml":"https://content.openalex.org/works/W4385963671.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3113091479","https://openalex.org/W2162899405","https://openalex.org/W941090075","https://openalex.org/W2044987316","https://openalex.org/W3134374554","https://openalex.org/W2237480245","https://openalex.org/W2075065631","https://openalex.org/W2519167559","https://openalex.org/W4387272257","https://openalex.org/W4200301313"],"abstract_inverted_index":{"Vision-language":[0],"models":[1],"like":[2],"CLIP":[3,91,109],"are":[4],"widely":[5],"used":[6],"in":[7],"zero-shot":[8,96,127],"image":[9,97],"classification":[10,98,128,145],"due":[11],"to":[12,15,26,34],"their":[13],"ability":[14],"understand":[16],"various":[17],"visual":[18,50],"concepts":[19],"and":[20,64,75,99,141,159],"natural":[21],"language":[22],"descriptions.":[23],"However,":[24],"how":[25],"fully":[27],"leverage":[28],"CLIP's":[29],"unprecedented":[30],"human-like":[31],"understanding":[32],"capabilities":[33],"achieve":[35],"better":[36,155],"performance":[37],"is":[38,163],"still":[39],"an":[40,55,117,132],"open":[41],"question.":[42],"This":[43],"paper":[44],"draws":[45],"inspiration":[46],"from":[47,72,116],"the":[48,69,73,78,114],"human":[49],"perception":[51],"process:":[52],"when":[53],"classifying":[54],"object,":[56],"humans":[57],"first":[58,135],"infer":[59,113],"contextual":[60,93,137],"attributes":[61,94,115,138],"(e.g.,":[62,139],"background":[63],"orientation)":[65],"which":[66],"help":[67],"separate":[68],"foreground":[70],"object":[71,79,144],"background,":[74],"then":[76,142],"classify":[77],"based":[80],"on":[81,102,147],"this":[82],"information.":[83],"Inspired":[84],"by":[85],"it,":[86],"we":[87,122],"observe":[88,107],"that":[89,108,152],"providing":[90],"with":[92],"improves":[95],"mitigates":[100],"reliance":[101],"spurious":[103],"features.":[104],"We":[105],"also":[106],"itself":[110],"can":[111],"reasonably":[112],"image.":[118],"With":[119],"these":[120],"observations,":[121],"propose":[123],"a":[124],"training-free,":[125],"two-step":[126],"method":[129],"PerceptionCLIP.":[130],"Given":[131],"image,":[133],"it":[134],"infers":[136],"background)":[140],"performs":[143],"conditioning":[146],"them.":[148],"Our":[149,161],"experiments":[150],"show":[151],"PerceptionCLIP":[153],"achieves":[154],"generalization,":[156],"group":[157],"robustness,":[158],"interoperability.":[160],"code":[162],"available":[164],"at":[165],"https://github.com/umd-huang-lab/perceptionCLIP":[166]},"counts_by_year":[],"updated_date":"2026-09-30T09:27:27.532063","created_date":"2023-08-18T00:00:00"}