{"id":"https://openalex.org/W4416323920","doi":"https://doi.org/10.1109/tpami.2025.3634165","title":"Synergy of Sight and Semantics: Holistic Visual Understanding With CLIP","display_name":"Synergy of Sight and Semantics: Holistic Visual Understanding With CLIP","publication_year":2025,"publication_date":"2025-11-18","ids":{"openalex":"https://openalex.org/W4416323920","doi":"https://doi.org/10.1109/tpami.2025.3634165","pmid":"https://pubmed.ncbi.nlm.nih.gov/41252232"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2025.3634165","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3634165","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101698068","display_name":"Qu Yang","orcid":"https://orcid.org/0009-0001-5397-0209"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qu Yang","raw_affiliation_strings":["School of Computer Science, Wuhan University, Wuhan, China","School of Computer Science, Wuhan University, China"],"raw_orcid":"https://orcid.org/0009-0001-5397-0209","affiliations":[{"raw_affiliation_string":"School of Computer Science, Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]},{"raw_affiliation_string":"School of Computer Science, Wuhan University, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008999954","display_name":"Mang Ye","orcid":"https://orcid.org/0000-0003-3989-7655"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mang Ye","raw_affiliation_strings":["School of Computer Science, Wuhan University, Wuhan, China","School of Computer Science, Wuhan University, China"],"raw_orcid":"https://orcid.org/0000-0003-3989-7655","affiliations":[{"raw_affiliation_string":"School of Computer Science, Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]},{"raw_affiliation_string":"School of Computer Science, Wuhan University, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074103823","display_name":"Dacheng Tao","orcid":"https://orcid.org/0000-0001-7225-5449"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Dacheng Tao","raw_affiliation_strings":["Generative AI Lab, College of Computing and Data Science, Nanyang Technological University, Singapore"],"raw_orcid":"https://orcid.org/0000-0001-7225-5449","affiliations":[{"raw_affiliation_string":"Generative AI Lab, College of Computing and Data Science, Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.31688745,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"48","issue":"3","first_page":"3119","last_page":"3136"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.8120999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.8120999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.030899999663233757,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.03060000017285347,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sight","display_name":"Sight","score":0.5802000164985657},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5153999924659729},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.5152000188827515},{"id":"https://openalex.org/keywords/visual-reasoning","display_name":"Visual reasoning","score":0.5013999938964844},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.47589999437332153},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.4546999931335449},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4219000041484833},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4104999899864197},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.39480000734329224}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8173999786376953},{"id":"https://openalex.org/C1517167","wikidata":"https://www.wikidata.org/wiki/Q1134322","display_name":"Sight","level":2,"score":0.5802000164985657},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5472000241279602},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5153999924659729},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.5152000188827515},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.5013999938964844},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.47589999437332153},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.4546999931335449},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4219000041484833},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4104999899864197},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.39480000734329224},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.39239999651908875},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.37959998846054077},{"id":"https://openalex.org/C178253425","wikidata":"https://www.wikidata.org/wiki/Q162668","display_name":"Visual perception","level":3,"score":0.3693000078201294},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.36890000104904175},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.3287999927997589},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.3174999952316284},{"id":"https://openalex.org/C198942812","wikidata":"https://www.wikidata.org/wiki/Q496618","display_name":"Semantic property","level":2,"score":0.3107999861240387},{"id":"https://openalex.org/C2781122975","wikidata":"https://www.wikidata.org/wiki/Q16928266","display_name":"Semantic feature","level":2,"score":0.30790001153945923},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3043999969959259},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.29350000619888306},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.2766000032424927},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2745000123977661},{"id":"https://openalex.org/C83665646","wikidata":"https://www.wikidata.org/wiki/Q42139305","display_name":"Feature vector","level":2,"score":0.27160000801086426},{"id":"https://openalex.org/C2776151529","wikidata":"https://www.wikidata.org/wiki/Q3045304","display_name":"Object detection","level":3,"score":0.26649999618530273},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.2655999958515167},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.25949999690055847}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2025.3634165","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3634165","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:41252232","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41252232","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2068814885","display_name":null,"funder_award_id":"62361166629","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8800375656","display_name":null,"funder_award_id":"62176188","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":44,"referenced_works":["https://openalex.org/W2017814585","https://openalex.org/W2108598243","https://openalex.org/W2152161678","https://openalex.org/W2194775991","https://openalex.org/W2347880541","https://openalex.org/W2517991028","https://openalex.org/W2732026016","https://openalex.org/W2962761264","https://openalex.org/W2963550079","https://openalex.org/W2964051877","https://openalex.org/W3008694996","https://openalex.org/W3042085267","https://openalex.org/W3097590906","https://openalex.org/W3098267758","https://openalex.org/W3149890733","https://openalex.org/W3176023283","https://openalex.org/W3198377975","https://openalex.org/W3198675127","https://openalex.org/W3211094633","https://openalex.org/W4205474609","https://openalex.org/W4205991051","https://openalex.org/W4206742551","https://openalex.org/W4214673031","https://openalex.org/W4225930680","https://openalex.org/W4312310776","https://openalex.org/W4313644168","https://openalex.org/W4321487955","https://openalex.org/W4362563637","https://openalex.org/W4382119137","https://openalex.org/W4385245566","https://openalex.org/W4386076678","https://openalex.org/W4386790226","https://openalex.org/W4390203943","https://openalex.org/W4390873312","https://openalex.org/W4393241415","https://openalex.org/W4394593185","https://openalex.org/W4403947066","https://openalex.org/W4405784827","https://openalex.org/W4408442637","https://openalex.org/W4409263215","https://openalex.org/W4409494545","https://openalex.org/W4409641172","https://openalex.org/W4410428697","https://openalex.org/W4413146243"],"related_works":[],"abstract_inverted_index":{"Holistic":[0],"Visual":[1,163],"Understanding":[2,57],"(HVU),":[3],"encompassing":[4],"tasks":[5],"like":[6,32],"intention":[7],"recognition,":[8],"emotion":[9],"analysis,":[10],"scene":[11],"understanding,":[12],"and":[13,75,113,144,162,192,198],"content":[14],"moderation,":[15],"necessitates":[16],"integrating":[17],"low-level":[18],"visual":[19,204],"perception":[20],"('sight')":[21],"with":[22,66],"high-level":[23],"semantic":[24,125,145],"reasoning":[25],"('semantics').":[26],"While":[27],"large":[28],"Vision-Language":[29],"Models":[30,111],"(VLMs)":[31],"CLIP":[33,118],"offer":[34],"powerful":[35],"representations,":[36],"their":[37,42],"inherent":[38],"'sight'":[39],"bias":[40],"limits":[41],"direct":[43],"application":[44],"to":[45,92,121],"these":[46],"semantically":[47],"rich":[48],"tasks.":[49,186],"Our":[50],"prior":[51],"work,":[52],"IntCLIP,":[53],"addressed":[54],"Multi-label":[55],"Intention":[56],"(MIU)":[58],"using":[59],"a":[60,130,151,190],"dual-branch":[61],"architecture":[62],"but":[63,178],"faced":[64],"challenges":[65],"label":[67],"generation":[68],"instability":[69],"(Hierarchical":[70],"Class":[71],"Integration":[72],"-":[73],"HCI)":[74],"limited":[76],"feature":[77,146],"interaction":[78],"(unidirectional":[79],"Sight-assisted":[80],"Aggregation).":[81],"This":[82,187],"paper":[83],"introduces":[84],"an":[85,104],"enhanced":[86],"framework":[87,170,188],"that":[88,137,168],"significantly":[89],"extends":[90],"IntCLIP":[91],"tackle":[93],"the":[94,117,174],"broader":[95],"HVU":[96,185],"challenge.":[97],"We":[98,127],"propose":[99],"Semantic":[100],"Label":[101],"Refinement":[102],"(SLR),":[103],"iterative,":[105],"metric-guided":[106],"process":[107],"leveraging":[108],"Large":[109],"Language":[110],"(LLMs)":[112],"quantitative":[114],"evaluation":[115],"within":[116],"embedding":[119],"space":[120],"generate":[122],"stable,":[123],"optimized":[124],"labels.":[126],"also":[128,179],"introduce":[129],"novel":[131],"bidirectional":[132],"attention":[133],"mechanism":[134],"(Symmetric":[135],"Aggregation)":[136],"enables":[138],"balanced,":[139],"mutual":[140],"refinement":[141],"between":[142],"sight":[143,197],"maps.":[147],"By":[148],"evaluating":[149],"on":[150],"comprehensive":[152],"benchmark":[153],"spanning":[154],"MIU,":[155],"Image":[156],"Emotion":[157],"Recognition,":[158,161],"Indoor":[159],"Scene":[160],"Content":[164],"Moderation,":[165],"we":[166],"demonstrate":[167],"our":[169],"not":[171],"only":[172],"advances":[173],"state-of-the-art":[175],"in":[176],"MIU":[177],"achieves":[180],"superior":[181],"performance":[182],"across":[183],"diverse":[184],"provides":[189],"unified":[191],"robust":[193],"solution":[194],"for":[195],"synergizing":[196],"semantics,":[199],"pushing":[200],"towards":[201],"more":[202],"human-like":[203],"intelligence.":[205],"Code":[206],"is":[207],"available":[208],"at":[209],"https://github.com/yan9qu/PAMI25-HVU.":[210]},"counts_by_year":[],"updated_date":"2026-02-07T06:11:34.122080","created_date":"2025-11-18T00:00:00"}