{"id":"https://openalex.org/W4385574166","doi":"https://doi.org/10.18653/v1/2022.nlp4pi-1.20","title":"Hate-CLIPper: Multimodal Hateful Meme Classification based on Cross-modal Interaction of CLIP Features","display_name":"Hate-CLIPper: Multimodal Hateful Meme Classification based on Cross-modal Interaction of CLIP Features","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4385574166","doi":"https://doi.org/10.18653/v1/2022.nlp4pi-1.20"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2022.nlp4pi-1.20","doi":"https://doi.org/10.18653/v1/2022.nlp4pi-1.20","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.nlp4pi-1.20","pdf_url":"https://aclanthology.org/2022.nlp4pi-1.20.pdf","source":{"id":"https://openalex.org/S7407106564","display_name":"Workshop on NLP for Positive Impact (NLP4PI)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Second Workshop on NLP for Positive Impact (NLP4PI)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2022.nlp4pi-1.20.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048139334","display_name":"Gokul Karthik Kumar","orcid":"https://orcid.org/0000-0003-2848-7095","observed_orcids":["https://orcid.org/0000-0003-2848-7095"]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":true,"raw_author_name":"Gokul Karthik Kumar","raw_affiliation_strings":["Zayed University of Artificial Intelligence (MBZUAI) Abu Dhabi, UAE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zayed University of Artificial Intelligence (MBZUAI) Abu Dhabi, UAE","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5059676452","display_name":"Karthik Nandakumar","orcid":"https://orcid.org/0000-0002-6274-9725","observed_orcids":["https://orcid.org/0000-0002-6274-9725"]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Karthik Nandakumar","raw_affiliation_strings":["Zayed University of Artificial Intelligence (MBZUAI) Abu Dhabi, UAE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zayed University of Artificial Intelligence (MBZUAI) Abu Dhabi, UAE","institution_ids":["https://openalex.org/I4210113480"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5048139334"],"corresponding_institution_ids":["https://openalex.org/I4210113480"],"apc_list":null,"apc_paid":null,"fwci":8.0087,"has_fulltext":true,"cited_by_count":110,"citation_normalized_percentile":{"value":0.9831383,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"171","last_page":"183"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/clip","display_name":"CLIP","score":0.6740000247955322},{"id":"https://openalex.org/keywords/cross-modal-interaction","display_name":"cross-modal interaction","score":0.8920000195503235},{"id":"https://openalex.org/keywords/image-text-fusion","display_name":"image-text fusion","score":0.8519999980926514},{"id":"https://openalex.org/keywords/social-media","display_name":"social media","score":0.38999998569488525},{"id":"https://openalex.org/keywords/interpretability","display_name":"interpretability","score":0.8849999904632568}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7566840648651123},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5369911193847656},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.4812176525592804},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.4794448912143707},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4345402717590332},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.42504578828811646},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3651066720485687},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2022.nlp4pi-1.20","doi":"https://doi.org/10.18653/v1/2022.nlp4pi-1.20","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.nlp4pi-1.20","pdf_url":"https://aclanthology.org/2022.nlp4pi-1.20.pdf","source":{"id":"https://openalex.org/S7407106564","display_name":"Workshop on NLP for Positive Impact (NLP4PI)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Second Workshop on NLP for Positive Impact (NLP4PI)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2210.05916","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2210.05916","pdf_url":"https://arxiv.org/pdf/2210.05916","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2210.05916","doi":"https://doi.org/10.48550/arxiv.2210.05916","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2210.05916","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2022.nlp4pi-1.20","doi":"https://doi.org/10.18653/v1/2022.nlp4pi-1.20","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.nlp4pi-1.20","pdf_url":"https://aclanthology.org/2022.nlp4pi-1.20.pdf","source":{"id":"https://openalex.org/S7407106564","display_name":"Workshop on NLP for Positive Impact (NLP4PI)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Second Workshop on NLP for Positive Impact (NLP4PI)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385574166.pdf","grobid_xml":"https://content.openalex.org/works/W4385574166.grobid-xml"},"referenced_works_count":40,"referenced_works":["https://openalex.org/W639708223","https://openalex.org/W1686810756","https://openalex.org/W2101234009","https://openalex.org/W2155653793","https://openalex.org/W2170653751","https://openalex.org/W2194775991","https://openalex.org/W2587372848","https://openalex.org/W2605982830","https://openalex.org/W2612445135","https://openalex.org/W2787873795","https://openalex.org/W2896457183","https://openalex.org/W2963383024","https://openalex.org/W2968124245","https://openalex.org/W2969876226","https://openalex.org/W2970608575","https://openalex.org/W2972119347","https://openalex.org/W2978017171","https://openalex.org/W3023989664","https://openalex.org/W3090449556","https://openalex.org/W3094502228","https://openalex.org/W3102995547","https://openalex.org/W3111930335","https://openalex.org/W3112525597","https://openalex.org/W3115116211","https://openalex.org/W3116547614","https://openalex.org/W3135367836","https://openalex.org/W3154552322","https://openalex.org/W3173909648","https://openalex.org/W3177048205","https://openalex.org/W3185758730","https://openalex.org/W3186646580","https://openalex.org/W3196454747","https://openalex.org/W3199394329","https://openalex.org/W3199938814","https://openalex.org/W4221166856","https://openalex.org/W4221167446","https://openalex.org/W4285606948","https://openalex.org/W4312910992","https://openalex.org/W4313164293","https://openalex.org/W4393711525"],"related_works":["https://openalex.org/W7126433569","https://openalex.org/W7131431341","https://openalex.org/W4390190020","https://openalex.org/W4411237556","https://openalex.org/W7202390104","https://openalex.org/W4404782199","https://openalex.org/W3175392088","https://openalex.org/W4401609024","https://openalex.org/W4415708402","https://openalex.org/W4407767103"],"abstract_inverted_index":{"Hateful":[0,145],"memes":[1,33],"are":[2,18],"a":[3,16,66,125],"growing":[4],"menace":[5],"on":[6,133,143,163],"social":[7],"media.While":[8],"the":[9,25,54,57,60,75,78,102,108,112,134,144,158,175,178,183,186,197],"image":[10,58,79,113],"and":[11,40,59,80,114,171,189],"its":[12],"corresponding":[13],"text":[14,61,81,115],"in":[15,65],"meme":[17,165],"related,":[19],"they":[20],"do":[21],"not":[22,96],"necessarily":[23],"convey":[24],"same":[26],"meaning":[27],"when":[28],"viewed":[29],"individually.Hence,":[30],"detecting":[31],"hateful":[32],"requires":[34],"careful":[35],"consideration":[36],"of":[37,153,161,177,185,199],"both":[38],"visual":[39],"textual":[41],"information.Multimodal":[42],"pretraining":[43],"can":[44,194],"be":[45],"beneficial":[46],"for":[47,203],"this":[48,98,204],"task":[49],"because":[50],"it":[51,70],"effectively":[52],"captures":[53],"relationship":[55],"between":[56,77,111],"by":[62],"representing":[63],"them":[64],"similar":[67],"feature":[68,126],"space.Furthermore,":[69],"is":[71,137,206],"essential":[72],"to":[73,139],"model":[74],"interactions":[76,110,193],"features":[82],"through":[83],"intermediate":[84,93],"fusion.Most":[85],"existing":[86],"methods":[87],"either":[88],"employ":[89],"multimodal":[90],"pre-training":[91],"or":[92],"fusion,":[94],"but":[95],"both.In":[97],"work,":[99],"we":[100,181],"propose":[101],"Hate-CLIPper":[103],"architecture,":[104],"which":[105,155],"explicitly":[106],"models":[107],"cross-modal":[109,192],"representations":[116],"obtained":[117],"using":[118],"Contrastive":[119],"Language-Image":[120],"Pre-training":[121],"(CLIP)":[122],"encoders":[123],"via":[124],"interaction":[127],"matrix":[128],"(FIM).A":[129],"simple":[130],"classifier":[131],"based":[132],"FIM":[135,187],"representation":[136,188],"able":[138],"achieve":[140],"state-of-the-art":[141],"performance":[142,160],"Memes":[146,170],"Challenge":[147],"(HMC)":[148],"dataset":[149],"with":[150],"an":[151],"AUROC":[152],"85.8,":[154],"even":[156],"surpasses":[157],"human":[159],"82.65.Experiments":[162],"other":[164],"datasets":[166],"such":[167],"as":[168],"Propaganda":[169],"TamilMemes":[172],"also":[173],"demonstrate":[174],"generalizability":[176],"proposed":[179],"approach.Finally,":[180],"analyze":[182],"interpretability":[184],"show":[190],"that":[191],"indeed":[195],"facilitate":[196],"learning":[198],"meaningful":[200],"concepts.The":[201],"code":[202],"work":[205],"available":[207],"at":[208],"https://github.com/gokulkarthik/hateclipper.":[209]},"counts_by_year":[{"year":2026,"cited_by_count":35},{"year":2025,"cited_by_count":38},{"year":2024,"cited_by_count":26},{"year":2023,"cited_by_count":11}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2023-08-05T00:00:00"}