{"id":"https://openalex.org/W4387969708","doi":"https://doi.org/10.1145/3581783.3612498","title":"Pro-Cap: Leveraging a Frozen Vision-Language Model for Hateful Meme Detection","display_name":"Pro-Cap: Leveraging a Frozen Vision-Language Model for Hateful Meme Detection","publication_year":2023,"publication_date":"2023-10-26","ids":{"openalex":"https://openalex.org/W4387969708","doi":"https://doi.org/10.1145/3581783.3612498"},"language":"en","primary_location":{"id":"doi:10.1145/3581783.3612498","doi":"https://doi.org/10.1145/3581783.3612498","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3581783.3612498","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3612498","source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Multimedia","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3612498","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077167782","display_name":"Rui Cao","orcid":"https://orcid.org/0000-0002-9444-6913","observed_orcids":["https://orcid.org/0000-0002-9444-6913","https://orcid.org/0009-0006-6182-1381"]},"institutions":[{"id":"https://openalex.org/I79891267","display_name":"Singapore Management University","ror":"https://ror.org/050qmg959","country_code":"SG","type":"education","lineage":["https://openalex.org/I79891267"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Rui Cao","raw_affiliation_strings":["Singapore Management University, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-9444-6913","affiliations":[{"raw_affiliation_string":"Singapore Management University, Singapore, Singapore","institution_ids":["https://openalex.org/I79891267"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045458136","display_name":"Ming Shan Hee","orcid":"https://orcid.org/0000-0002-6328-5889","observed_orcids":["https://orcid.org/0000-0002-6328-5889"]},"institutions":[{"id":"https://openalex.org/I152815399","display_name":"Singapore University of Technology and Design","ror":"https://ror.org/05j6fvn87","country_code":"SG","type":"education","lineage":["https://openalex.org/I152815399"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Ming Shan Hee","raw_affiliation_strings":["Singapore University of Design and Technology, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-6328-5889","affiliations":[{"raw_affiliation_string":"Singapore University of Design and Technology, Singapore, Singapore","institution_ids":["https://openalex.org/I152815399"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092838055","display_name":"Adriel Kuek","orcid":"https://orcid.org/0009-0009-5190-2534","observed_orcids":["https://orcid.org/0009-0009-5190-2534"]},"institutions":[{"id":"https://openalex.org/I28490864","display_name":"DSO National Laboratories","ror":"https://ror.org/03e05fb06","country_code":"SG","type":"nonprofit","lineage":["https://openalex.org/I28490864"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Adriel Kuek","raw_affiliation_strings":["DSO National Laboratories, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0009-0009-5190-2534","affiliations":[{"raw_affiliation_string":"DSO National Laboratories, Singapore, Singapore","institution_ids":["https://openalex.org/I28490864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074674139","display_name":"Wen-Haw Chong","orcid":"https://orcid.org/0009-0000-7189-8370","observed_orcids":["https://orcid.org/0009-0000-7189-8370"]},"institutions":[{"id":"https://openalex.org/I79891267","display_name":"Singapore Management University","ror":"https://ror.org/050qmg959","country_code":"SG","type":"education","lineage":["https://openalex.org/I79891267"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Wen-Haw Chong","raw_affiliation_strings":["Singapore Management University, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0009-0000-7189-8370","affiliations":[{"raw_affiliation_string":"Singapore Management University, Singapore, Singapore","institution_ids":["https://openalex.org/I79891267"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089793938","display_name":"Roy Ka-Wei Lee","orcid":"https://orcid.org/0000-0002-1986-7750","observed_orcids":["https://orcid.org/0000-0002-1986-7750"]},"institutions":[{"id":"https://openalex.org/I152815399","display_name":"Singapore University of Technology and Design","ror":"https://ror.org/05j6fvn87","country_code":"SG","type":"education","lineage":["https://openalex.org/I152815399"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Roy Ka-Wei Lee","raw_affiliation_strings":["Singapore University of Design and Technology, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-1986-7750","affiliations":[{"raw_affiliation_string":"Singapore University of Design and Technology, Singapore, Singapore","institution_ids":["https://openalex.org/I152815399"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024040521","display_name":"Jing Jiang","orcid":"https://orcid.org/0000-0002-3035-0074","observed_orcids":["https://orcid.org/0000-0002-3035-0074","https://orcid.org/0000-0001-6715-2875"]},"institutions":[{"id":"https://openalex.org/I79891267","display_name":"Singapore Management University","ror":"https://ror.org/050qmg959","country_code":"SG","type":"education","lineage":["https://openalex.org/I79891267"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Jing Jiang","raw_affiliation_strings":["Singapore Management University, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-3035-0074","affiliations":[{"raw_affiliation_string":"Singapore Management University, Singapore, Singapore","institution_ids":["https://openalex.org/I79891267"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.6168,"has_fulltext":true,"cited_by_count":78,"citation_normalized_percentile":{"value":0.98478709,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"5244","last_page":"5252"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9775000214576721,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9775000214576721,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0210999995470047,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.8119999766349792},{"id":"https://openalex.org/keywords/frozen-pre-trained-models","display_name":"frozen pre-trained models","score":0.7570000290870667},{"id":"https://openalex.org/keywords/image-captioning","display_name":"image captioning","score":0.8309999704360962}],"concepts":[{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.9187301397323608},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.8137099742889404},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8109203577041626},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.6286479234695435},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5911123156547546},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5737497806549072},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5677098035812378},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.5230315923690796},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.4898640811443329},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3533475995063782},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3401006758213043},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.3254357874393463},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3581783.3612498","doi":"https://doi.org/10.1145/3581783.3612498","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3581783.3612498","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3612498","source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Multimedia","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2308.08088","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.08088","pdf_url":"https://arxiv.org/pdf/2308.08088","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:ink.library.smu.edu.sg:sis_research-9480","doi":null,"is_oa":true,"landing_page_url":"https://ink.library.smu.edu.sg/sis_research/8477","pdf_url":null,"source":{"id":"https://openalex.org/S4306401925","display_name":"Singapore Management University Institutional Knowledge (InK) (Singapore Management University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I79891267","host_organization_name":"Singapore Management University","host_organization_lineage":["https://openalex.org/I79891267"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"https://doi.org/10.1145/3581783.3612498","raw_type":"Conference Proceeding Article"},{"id":"doi:10.48550/arxiv.2308.08088","doi":"https://doi.org/10.48550/arxiv.2308.08088","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2308.08088","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1145/3581783.3612498","doi":"https://doi.org/10.1145/3581783.3612498","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3581783.3612498","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3612498","source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Multimedia","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387969708.pdf","grobid_xml":"https://content.openalex.org/works/W4387969708.grobid-xml"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W639708223","https://openalex.org/W2886641317","https://openalex.org/W2908510526","https://openalex.org/W2962858109","https://openalex.org/W2965373594","https://openalex.org/W2968124245","https://openalex.org/W2970608575","https://openalex.org/W3111930335","https://openalex.org/W3112525597","https://openalex.org/W3115116211","https://openalex.org/W3118051320","https://openalex.org/W3173777717","https://openalex.org/W3174604160","https://openalex.org/W3175392088","https://openalex.org/W3185758730","https://openalex.org/W3211483028","https://openalex.org/W3214040319","https://openalex.org/W4224320265","https://openalex.org/W4225323055","https://openalex.org/W4283455941","https://openalex.org/W4287855069","https://openalex.org/W4318718936","https://openalex.org/W4322718191","https://openalex.org/W4385573042","https://openalex.org/W4385767874","https://openalex.org/W4386566667","https://openalex.org/W6600159499","https://openalex.org/W6600234944","https://openalex.org/W6600424091"],"related_works":["https://openalex.org/W4411120736","https://openalex.org/W6947946343","https://openalex.org/W4295425581","https://openalex.org/W7221086290","https://openalex.org/W3200681734","https://openalex.org/W7204591044","https://openalex.org/W4388684938","https://openalex.org/W4396723445","https://openalex.org/W3175392088","https://openalex.org/W4220994850"],"abstract_inverted_index":{"Hateful":[0],"meme":[1,59],"detection":[2],"is":[3],"a":[4,90,98,108],"challenging":[5],"multimodal":[6],"task":[7],"that":[8,128],"requires":[9],"comprehension":[10],"of":[11,141,153],"both":[12],"vision":[13],"and":[14,64,116,151],"language,":[15],"as":[16,18,120],"well":[17],"cross-modal":[19],"interactions.":[20],"Recent":[21],"studies":[22],"have":[23,55],"tried":[24],"to":[25,42,57,94],"fine-tune":[26],"pre-trained":[27],"vision-language":[28],"models":[29,67,142],"(PVLMs)":[30],"for":[31,68,134],"this":[32],"task.":[33],"However,":[34],"with":[35,143],"increasing":[36],"model":[37],"sizes,":[38],"it":[39],"becomes":[40],"important":[41],"leverage":[43,95],"powerful":[44],"PVLMs":[45,96],"more":[46],"efficiently,":[47],"rather":[48],"than":[49],"simply":[50],"fine-tuning":[51],"them.":[52],"Recently,":[53],"researchers":[54],"attempted":[56],"convert":[58],"images":[60],"into":[61],"textual":[62],"captions":[63,122,130],"prompt":[65,107],"language":[66],"predictions.":[69],"This":[70],"approach":[71,93],"has":[72],"shown":[73],"good":[74,139],"performance":[75,140],"but":[76],"suffers":[77],"from":[78],"non-informative":[79],"image":[80,121],"captions.":[81],"Considering":[82],"the":[83,118,129,149,154],"two":[84],"factors":[85],"mentioned":[86],"above,":[87],"we":[88,106,124],"propose":[89],"probing-based":[91],"captioning":[92],"in":[97],"zero-shot":[99],"visual":[100],"question":[101],"answering":[102],"(VQA)":[103],"manner.":[104],"Specifically,":[105],"frozen":[109],"PVLM":[110],"by":[111],"asking":[112],"hateful":[113,135],"content-related":[114],"questions":[115],"use":[117],"answers":[119],"(which":[123],"call":[125],"Pro-Cap),":[126],"so":[127],"contain":[131],"information":[132],"critical":[133],"content":[136],"detection.":[137],"The":[138],"Pro-Cap":[144],"on":[145],"three":[146],"benchmarks":[147],"validates":[148],"effectiveness":[150],"generalization":[152],"proposed":[155],"method1.":[156]},"counts_by_year":[{"year":2026,"cited_by_count":36},{"year":2025,"cited_by_count":25},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-10-28T00:00:00"}