{"id":"https://openalex.org/W4389317971","doi":"https://doi.org/10.1109/tnnls.2023.3335859","title":"ProtoCLIP: Prototypical Contrastive Language Image Pretraining","display_name":"ProtoCLIP: Prototypical Contrastive Language Image Pretraining","publication_year":2023,"publication_date":"2023-12-04","ids":{"openalex":"https://openalex.org/W4389317971","doi":"https://doi.org/10.1109/tnnls.2023.3335859","pmid":"https://pubmed.ncbi.nlm.nih.gov/38048244"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2023.3335859","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2023.3335859","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020620488","display_name":"Delong Chen","orcid":"https://orcid.org/0000-0002-8172-2894"},"institutions":[{"id":"https://openalex.org/I163340411","display_name":"Hohai University","ror":"https://ror.org/01wd4xt90","country_code":"CN","type":"education","lineage":["https://openalex.org/I163340411"]},{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Delong Chen","raw_affiliation_strings":["College of Computer and Information, Hohai University, Nanjing, China","Centre for Artificial Intelligence Research (CAiRE), The Hong Kong University of Science and Technology (HKUST), Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0002-8172-2894","affiliations":[{"raw_affiliation_string":"College of Computer and Information, Hohai University, Nanjing, China","institution_ids":["https://openalex.org/I163340411"]},{"raw_affiliation_string":"Centre for Artificial Intelligence Research (CAiRE), The Hong Kong University of Science and Technology (HKUST), Hong Kong, China","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100409239","display_name":"Wu Zhao","orcid":"https://orcid.org/0000-0001-8774-4894"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao Wu","raw_affiliation_strings":["MEGVII Research, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MEGVII Research, Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100371024","display_name":"Fan Liu","orcid":"https://orcid.org/0000-0001-8746-9845"},"institutions":[{"id":"https://openalex.org/I163340411","display_name":"Hohai University","ror":"https://ror.org/01wd4xt90","country_code":"CN","type":"education","lineage":["https://openalex.org/I163340411"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fan Liu","raw_affiliation_strings":["College of Computer and Information, Hohai University, Nanjing, China","Science and Technology on Underwater Vehicle Technology Laboratory, Harbin, China"],"raw_orcid":"https://orcid.org/0000-0001-8746-9845","affiliations":[{"raw_affiliation_string":"College of Computer and Information, Hohai University, Nanjing, China","institution_ids":["https://openalex.org/I163340411"]},{"raw_affiliation_string":"Science and Technology on Underwater Vehicle Technology Laboratory, Harbin, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089671682","display_name":"Zaiquan Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zaiquan Yang","raw_affiliation_strings":["MEGVII Research, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MEGVII Research, Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083317426","display_name":"Shaoqiu Zheng","orcid":null},"institutions":[{"id":"https://openalex.org/I4210110458","display_name":"Institute of Electronics","ror":"https://ror.org/01z143507","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210110458"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaoqiu Zheng","raw_affiliation_strings":["Nanjing Research Institute of Electronic Engineering, Nanjing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanjing Research Institute of Electronic Engineering, Nanjing, China","institution_ids":["https://openalex.org/I4210110458"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023089209","display_name":"Ying Tan","orcid":"https://orcid.org/0000-0001-8243-4731"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ying Tan","raw_affiliation_strings":["Department of Machine Intelligence, School of EECS, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8243-4731","affiliations":[{"raw_affiliation_string":"Department of Machine Intelligence, School of EECS, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5060050139","display_name":"Erjin Zhou","orcid":"https://orcid.org/0000-0001-9234-0026"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Erjin Zhou","raw_affiliation_strings":["MEGVII Research, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MEGVII Research, Beijing, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.2902,"has_fulltext":false,"cited_by_count":64,"citation_normalized_percentile":{"value":0.97894146,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"36","issue":"1","first_page":"610","last_page":"624"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12151","display_name":"Interpreting and Communication in Healthcare","score":0.9739000201225281,"subfield":{"id":"https://openalex.org/subfields/3600","display_name":"General Health Professions"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7551384568214417},{"id":"https://openalex.org/keywords/boosting","display_name":"Boosting (machine learning)","score":0.7236248254776001},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.593358039855957},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.5913974046707153},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.57203209400177},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.45125943422317505},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3300687074661255},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.32093125581741333}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7551384568214417},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.7236248254776001},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.593358039855957},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5913974046707153},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.57203209400177},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.45125943422317505},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3300687074661255},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.32093125581741333},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tnnls.2023.3335859","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2023.3335859","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:38048244","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/38048244","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-133855","is_oa":false,"landing_page_url":"https://repository.hkust.edu.hk/ir/Record/1783.1-133855","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5099999904632568,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"},{"score":0.41999998688697815,"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10"}],"awards":[{"id":"https://openalex.org/G7400425163","display_name":null,"funder_award_id":"62372155","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8009129191","display_name":null,"funder_award_id":"2021063","funder_id":"https://openalex.org/F4320327794","funder_display_name":"Qinglan Project of Jiangsu Province of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320327794","display_name":"Qinglan Project of Jiangsu Province of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":72,"referenced_works":["https://openalex.org/W12634471","https://openalex.org/W1821462560","https://openalex.org/W1977295328","https://openalex.org/W1993309459","https://openalex.org/W2047643928","https://openalex.org/W2108598243","https://openalex.org/W2117876524","https://openalex.org/W2138011018","https://openalex.org/W2194775991","https://openalex.org/W2250384498","https://openalex.org/W2533598788","https://openalex.org/W2806070179","https://openalex.org/W2842511635","https://openalex.org/W2883725317","https://openalex.org/W2886641317","https://openalex.org/W2896457183","https://openalex.org/W2963140444","https://openalex.org/W2963855133","https://openalex.org/W2964005298","https://openalex.org/W2965373594","https://openalex.org/W2998702515","https://openalex.org/W3023371261","https://openalex.org/W3035524453","https://openalex.org/W3090449556","https://openalex.org/W3091588028","https://openalex.org/W3158714121","https://openalex.org/W3158986867","https://openalex.org/W3159481202","https://openalex.org/W3168997536","https://openalex.org/W3169827396","https://openalex.org/W3171007011","https://openalex.org/W3171153522","https://openalex.org/W3171632293","https://openalex.org/W3205071530","https://openalex.org/W4221166941","https://openalex.org/W4226512186","https://openalex.org/W4285145317","https://openalex.org/W4297697565","https://openalex.org/W4313136445","https://openalex.org/W4385245566","https://openalex.org/W4385944963","https://openalex.org/W4393157162","https://openalex.org/W6631190155","https://openalex.org/W6639432524","https://openalex.org/W6726497184","https://openalex.org/W6745245109","https://openalex.org/W6757817989","https://openalex.org/W6758857762","https://openalex.org/W6766904570","https://openalex.org/W6767279747","https://openalex.org/W6770196601","https://openalex.org/W6770805772","https://openalex.org/W6777265123","https://openalex.org/W6778102432","https://openalex.org/W6779844698","https://openalex.org/W6779997284","https://openalex.org/W6790672419","https://openalex.org/W6791353385","https://openalex.org/W6797255657","https://openalex.org/W6798132802","https://openalex.org/W6801730907","https://openalex.org/W6803510752","https://openalex.org/W6803728867","https://openalex.org/W6803764725","https://openalex.org/W6803953248","https://openalex.org/W6804182625","https://openalex.org/W6810080845","https://openalex.org/W6810362786","https://openalex.org/W6810452849","https://openalex.org/W6810837128","https://openalex.org/W6839761088","https://openalex.org/W6853528422"],"related_works":["https://openalex.org/W2125652721","https://openalex.org/W1540371141","https://openalex.org/W1549363203","https://openalex.org/W2154063878","https://openalex.org/W4231274751","https://openalex.org/W2556012038","https://openalex.org/W1489772951","https://openalex.org/W1538046993","https://openalex.org/W3082059448","https://openalex.org/W4313640622"],"abstract_inverted_index":{"Contrastive":[0],"language":[1,73,152],"image":[2,74,102],"pretraining":[3,75],"(CLIP)":[4],"has":[5],"received":[6],"widespread":[7],"attention":[8],"since":[9],"its":[10,85,89],"learned":[11],"representations":[12,57,133],"can":[13,166],"be":[14,167],"transferred":[15],"well":[16],"to":[17,79,120,144,170],"various":[18],"downstream":[19],"tasks.":[20],"During":[21],"the":[22,26,29,50,92,199,205],"training":[23,161,212],"process":[24],"of":[25,131,173,207,211],"CLIP":[27,208],"model,":[28],"InfoNCE":[30,51],"objective":[31,52],"aligns":[32],"positive":[33],"image-text":[34],"pairs":[35],"and":[36,87,103,183,191],"separates":[37],"negative":[38],"ones.":[39],"We":[40,175],"show":[41],"an":[42,158,185,192],"underlying":[43],"representation":[44,122,125],"grouping":[45,82,123],"effect":[46],"during":[47],"this":[48,66,69],"process:":[49],"indirectly":[53],"groups":[54],"semantically":[55],"similar":[56],"together":[58],"via":[59],"randomly":[60],"emerged":[61],"within-modal":[62],"anchors.":[63],"Based":[64],"on":[65,179],"understanding,":[67],"in":[68,128],"article,":[70],"prototypical":[71,114],"contrastive":[72],"(ProtoCLIP)":[76],"is":[77,118,155],"introduced":[78],"enhance":[80],"such":[81],"by":[83],"boosting":[84],"efficiency":[86],"increasing":[88],"robustness":[90],"against":[91],"modality":[93,137],"gap.":[94,138],"Specifically,":[95],"ProtoCLIP":[96,154,178,203],"sets":[97],"up":[98,169],"prototype-level":[99],"discrimination":[100],"between":[101],"text":[104],"spaces,":[105],"which":[106,163],"efficiently":[107],"transfers":[108],"higher":[109],"level":[110],"structural":[111],"knowledge.":[112,153],"Furthermore,":[113],"back":[115],"translation":[116],"(PBT)":[117],"proposed":[119],"decouple":[121],"from":[124],"alignment,":[126],"resulting":[127],"effective":[129],"learning":[130],"meaningful":[132],"under":[134],"a":[135],"large":[136],"The":[139],"PBT":[140],"also":[141],"enables":[142],"us":[143],"introduce":[145],"additional":[146],"external":[147],"teachers":[148],"with":[149,157,209],"richer":[150],"prior":[151],"trained":[156,176],"online":[159],"episodic":[160],"strategy,":[162],"means":[164],"it":[165],"scaled":[168],"unlimited":[171],"amounts":[172],"data.":[174],"our":[177],"conceptual":[180],"captions":[181],"(CCs)":[182],"achieved":[184],"+5.81%":[186],"ImageNet":[187,194],"linear":[188],"probing":[189],"improvement":[190],"+2.01%":[193],"zero-shot":[195],"classification":[196],"improvement.":[197],"On":[198],"larger":[200],"YFCC-15M":[201],"dataset,":[202],"matches":[204],"performance":[206],"33%":[210],"time.":[213]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":42},{"year":2023,"cited_by_count":6}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}