{"id":"https://openalex.org/W4312402389","doi":"https://doi.org/10.1109/mmsp55362.2022.9949462","title":"Multimodal Semantic Mismatch Detection in Social Media Posts","display_name":"Multimodal Semantic Mismatch Detection in Social Media Posts","publication_year":2022,"publication_date":"2022-09-26","ids":{"openalex":"https://openalex.org/W4312402389","doi":"https://doi.org/10.1109/mmsp55362.2022.9949462"},"language":"en","primary_location":{"id":"doi:10.1109/mmsp55362.2022.9949462","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mmsp55362.2022.9949462","pdf_url":null,"source":{"id":"https://openalex.org/S4363605768","display_name":"2022 IEEE 24th International Workshop on Multimedia Signal Processing (MMSP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE 24th International Workshop on Multimedia Signal Processing (MMSP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024060116","display_name":"Kehan Wang","orcid":"https://orcid.org/0000-0001-8573-5265"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Kehan Wang","raw_affiliation_strings":["University of California, Berkeley,Berkeley,USA","University of California, Berkeley, Berkeley, USA"],"affiliations":[{"raw_affiliation_string":"University of California, Berkeley,Berkeley,USA","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"University of California, Berkeley, Berkeley, USA","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007649701","display_name":"Seth Z. Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Seth Z. Zhao","raw_affiliation_strings":["University of California, Berkeley,Berkeley,USA","University of California, Berkeley, Berkeley, USA"],"affiliations":[{"raw_affiliation_string":"University of California, Berkeley,Berkeley,USA","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"University of California, Berkeley, Berkeley, USA","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102804009","display_name":"David W. Chan","orcid":"https://orcid.org/0000-0003-4743-0967"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Chan","raw_affiliation_strings":["University of California, Berkeley,Berkeley,USA","University of California, Berkeley, Berkeley, USA"],"affiliations":[{"raw_affiliation_string":"University of California, Berkeley,Berkeley,USA","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"University of California, Berkeley, Berkeley, USA","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008034417","display_name":"Avideh Zakhor","orcid":"https://orcid.org/0000-0003-4770-6353"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Avideh Zakhor","raw_affiliation_strings":["University of California, Berkeley,Berkeley,USA","University of California, Berkeley, Berkeley, USA"],"affiliations":[{"raw_affiliation_string":"University of California, Berkeley,Berkeley,USA","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"University of California, Berkeley, Berkeley, USA","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5089723214","display_name":"John Canny","orcid":"https://orcid.org/0000-0002-7161-7927"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Canny","raw_affiliation_strings":["University of California, Berkeley,Berkeley,USA","University of California, Berkeley, Berkeley, USA"],"affiliations":[{"raw_affiliation_string":"University of California, Berkeley,Berkeley,USA","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"University of California, Berkeley, Berkeley, USA","institution_ids":["https://openalex.org/I95457486"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5024060116"],"corresponding_institution_ids":["https://openalex.org/I95457486"],"apc_list":null,"apc_paid":null,"fwci":0.3118,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.52095454,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7285816073417664},{"id":"https://openalex.org/keywords/social-media","display_name":"Social media","score":0.6277649402618408},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.40474340319633484},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3557016849517822},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3282257318496704},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2842966914176941}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7285816073417664},{"id":"https://openalex.org/C518677369","wikidata":"https://www.wikidata.org/wiki/Q202833","display_name":"Social media","level":2,"score":0.6277649402618408},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.40474340319633484},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3557016849517822},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3282257318496704},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2842966914176941}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/mmsp55362.2022.9949462","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mmsp55362.2022.9949462","pdf_url":null,"source":{"id":"https://openalex.org/S4363605768","display_name":"2022 IEEE 24th International Workshop on Multimedia Signal Processing (MMSP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE 24th International Workshop on Multimedia Signal Processing (MMSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.699999988079071,"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W2157364932","https://openalex.org/W2890930935","https://openalex.org/W2994071599","https://openalex.org/W3005680577","https://openalex.org/W3009561768","https://openalex.org/W3035635319","https://openalex.org/W3036601975","https://openalex.org/W3102409145","https://openalex.org/W3156355701","https://openalex.org/W3164804577","https://openalex.org/W3166396011","https://openalex.org/W3174408325","https://openalex.org/W3186163009","https://openalex.org/W3211686893","https://openalex.org/W4205602376","https://openalex.org/W4297808394","https://openalex.org/W4385245566","https://openalex.org/W6739901393","https://openalex.org/W6774314701","https://openalex.org/W6791353385","https://openalex.org/W6795778914","https://openalex.org/W6799600525","https://openalex.org/W6803557570"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2478288626","https://openalex.org/W2350741829","https://openalex.org/W2530322880","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Short":[0],"videos":[1],"have":[2],"become":[3],"the":[4,20,35,94,100,121],"most":[5],"popular":[6],"form":[7],"of":[8,80,96,123],"social":[9],"media":[10],"in":[11,116,144],"recent":[12],"years.":[13],"In":[14],"this":[15],"work,":[16],"we":[17],"focus":[18],"on":[19,77,126],"threat":[21],"scenario":[22],"where":[23],"video,":[24,49],"audio,":[25],"and":[26,51,58,68,99,131,135],"their":[27],"text":[28],"description":[29],"are":[30],"semantically":[31],"mismatched":[32],"to":[33,41,61,87],"mislead":[34],"audience.":[36],"We":[37,53,118],"develop":[38],"self-supervised":[39],"methods":[40,76],"detect":[42],"semantic":[43],"mismatch":[44],"across":[45],"multiple":[46],"modalities,":[47],"namely":[48],"audio":[50,59],"text.":[52],"use":[54],"state-of-the-art":[55,112,140],"language,":[56],"video":[57],"models":[60],"extract":[62],"dense":[63],"features":[64],"from":[65,85,93],"each":[66],"modality,":[67],"explore":[69],"transformer":[70],"architecture":[71],"together":[72,106],"with":[73],"contrastive":[74],"learning":[75],"a":[78,108],"dataset":[79],"one":[81],"million":[82],"Twitter":[83],"posts":[84],"2021":[86],"2022.":[88],"Our":[89],"best-performing":[90],"method":[91],"benefits":[92],"robustness":[95],"Noise-Contrastive":[97],"loss":[98],"context":[101],"provided":[102],"by":[103,113,141],"fusing":[104],"modalities":[105],"using":[107],"cross-transformer.":[109],"It":[110],"outperforms":[111,139],"over":[114,142],"9%":[115],"accuracy.":[117,145],"further":[119],"characterize":[120],"performance":[122],"our":[124],"system":[125],"topic-specific":[127],"datasets":[128],"containing":[129],"COVID-19":[130],"Russia-Ukraine":[132],"related":[133],"tweets,":[134],"shows":[136],"that":[137],"it":[138],"17%":[143]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}