{"id":"https://openalex.org/W4391952560","doi":"https://doi.org/10.1109/tpami.2024.3367749","title":"Detecting and Grounding Multi-Modal Media Manipulation and Beyond","display_name":"Detecting and Grounding Multi-Modal Media Manipulation and Beyond","publication_year":2024,"publication_date":"2024-02-20","ids":{"openalex":"https://openalex.org/W4391952560","doi":"https://doi.org/10.1109/tpami.2024.3367749","pmid":"https://pubmed.ncbi.nlm.nih.gov/38376967"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2024.3367749","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2024.3367749","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037878863","display_name":"Rui Shao","orcid":"https://orcid.org/0000-0003-0090-9604"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rui Shao","raw_affiliation_strings":["School of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China","School of Computer Science and Technology, Harbin Institute of Technology (Shenzhen), China"],"raw_orcid":"https://orcid.org/0000-0003-0090-9604","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]},{"raw_affiliation_string":"School of Computer Science and Technology, Harbin Institute of Technology (Shenzhen), China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102232690","display_name":"Tianxing Wu","orcid":"https://orcid.org/0000-0001-7345-0254"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Tianxing Wu","raw_affiliation_strings":["S-Lab, Nanyang Technological University, Singapore"],"raw_orcid":"https://orcid.org/0000-0001-7345-0254","affiliations":[{"raw_affiliation_string":"S-Lab, Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100654190","display_name":"Jianlong Wu","orcid":"https://orcid.org/0000-0003-0247-5221"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianlong Wu","raw_affiliation_strings":["School of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China","School of Computer Science and Technology, Harbin Institute of Technology (Shenzhen), China"],"raw_orcid":"https://orcid.org/0000-0003-0247-5221","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]},{"raw_affiliation_string":"School of Computer Science and Technology, Harbin Institute of Technology (Shenzhen), China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038612499","display_name":"Liqiang Nie","orcid":"https://orcid.org/0000-0003-1476-0273"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liqiang Nie","raw_affiliation_strings":["School of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China","School of Computer Science and Technology, Harbin Institute of Technology (Shenzhen), China"],"raw_orcid":"https://orcid.org/0000-0003-1476-0273","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]},{"raw_affiliation_string":"School of Computer Science and Technology, Harbin Institute of Technology (Shenzhen), China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100406050","display_name":"Ziwei Liu","orcid":"https://orcid.org/0000-0002-4220-5958"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Ziwei Liu","raw_affiliation_strings":["S-Lab, Nanyang Technological University, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-4220-5958","affiliations":[{"raw_affiliation_string":"S-Lab, Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.1554,"has_fulltext":false,"cited_by_count":43,"citation_normalized_percentile":{"value":0.98270947,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"46","issue":"8","first_page":"5556","last_page":"5574"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12357","display_name":"Digital Media Forensic Detection","score":0.9876000285148621,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6642919778823853},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.5688251852989197},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.56875079870224},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.39889249205589294}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6642919778823853},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.5688251852989197},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.56875079870224},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.39889249205589294},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2024.3367749","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2024.3367749","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:38376967","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/38376967","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5630254371","display_name":null,"funder_award_id":"62236003","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5954055007","display_name":null,"funder_award_id":"62376069","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5972905494","display_name":null,"funder_award_id":"62306090","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320320751","display_name":"Ministry of Education - Singapore","ror":"https://ror.org/01kcva023"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":78,"referenced_works":["https://openalex.org/W2341528187","https://openalex.org/W2742330194","https://openalex.org/W2766462585","https://openalex.org/W2773666902","https://openalex.org/W2798405286","https://openalex.org/W2809476703","https://openalex.org/W2889058905","https://openalex.org/W2896457183","https://openalex.org/W2908510526","https://openalex.org/W2912305564","https://openalex.org/W2951299559","https://openalex.org/W2952335829","https://openalex.org/W2952476201","https://openalex.org/W2962766617","https://openalex.org/W2963667126","https://openalex.org/W2969792713","https://openalex.org/W2970562804","https://openalex.org/W2970641574","https://openalex.org/W2982058372","https://openalex.org/W2987809008","https://openalex.org/W3034196597","https://openalex.org/W3034713808","https://openalex.org/W3034900344","https://openalex.org/W3035524453","https://openalex.org/W3092709185","https://openalex.org/W3094502228","https://openalex.org/W3094728142","https://openalex.org/W3099215402","https://openalex.org/W3101998545","https://openalex.org/W3104415840","https://openalex.org/W3140110584","https://openalex.org/W3156355701","https://openalex.org/W3158353280","https://openalex.org/W3174508664","https://openalex.org/W3174656926","https://openalex.org/W3175342695","https://openalex.org/W3176241004","https://openalex.org/W3176990204","https://openalex.org/W3183392865","https://openalex.org/W3183999072","https://openalex.org/W3211405508","https://openalex.org/W3215028386","https://openalex.org/W4214680478","https://openalex.org/W4214926101","https://openalex.org/W4220742835","https://openalex.org/W4285159427","https://openalex.org/W4286976841","https://openalex.org/W4287388155","https://openalex.org/W4287758545","https://openalex.org/W4288090950","https://openalex.org/W4295887562","https://openalex.org/W4296554086","https://openalex.org/W4297808394","https://openalex.org/W4300952844","https://openalex.org/W4312350393","https://openalex.org/W4312544276","https://openalex.org/W4312877428","https://openalex.org/W4385245566","https://openalex.org/W4386071472","https://openalex.org/W4386075867","https://openalex.org/W6637373629","https://openalex.org/W6745560452","https://openalex.org/W6752788575","https://openalex.org/W6757817989","https://openalex.org/W6762900673","https://openalex.org/W6763240421","https://openalex.org/W6769571810","https://openalex.org/W6769705151","https://openalex.org/W6769920582","https://openalex.org/W6783570321","https://openalex.org/W6784810453","https://openalex.org/W6785462510","https://openalex.org/W6789038945","https://openalex.org/W6789753369","https://openalex.org/W6791353385","https://openalex.org/W6798805250","https://openalex.org/W6804095316","https://openalex.org/W6844194202"],"related_works":["https://openalex.org/W2772917594","https://openalex.org/W2036807459","https://openalex.org/W2058170566","https://openalex.org/W2755342338","https://openalex.org/W2166024367","https://openalex.org/W3116076068","https://openalex.org/W2229312674","https://openalex.org/W2951359407","https://openalex.org/W2079911747","https://openalex.org/W1969923398"],"abstract_inverted_index":{"Misinformation":[0],"has":[1],"become":[2],"a":[3,58,125,137,240],"pressing":[4],"issue.":[5],"Fake":[6],"media,":[7,65,108],"in":[8,288],"both":[9],"visual":[10],"and":[11,22,45,70,189,203,238,253,273],"textual":[12],"forms,":[13],"is":[14],"widespread":[15],"on":[16,39,214],"the":[17,104,112,130,169,215,267],"web.":[18],"Whilevarious":[19],"deepfake":[20],"detection":[21,26,202],"text":[23],"fake":[24,64],"news":[25],"methods":[27],"have":[28],"been":[29],"proposed,":[30],"they":[31],"are":[32,206,281],"only":[33,102],"designed":[34],"for":[35,62,225,259],"single-modality":[36],"forgery":[37,48],"based":[38,213],"binary":[40],"classification,":[41],"let":[42],"alone":[43],"analyzing":[44],"reasoning":[46,118],"subtle":[47],"traces":[49],"across":[50],"different":[51,173],"modalities.":[52,174],"In":[53],"this":[54,260],"paper,":[55],"we":[56,128,135,229,248],"highlight":[57],"new":[59,261],"research":[60,262,287],"problem":[61],"multi-modal":[63,107,120,194,217,289],"namely":[66],"<bold":[67,71,75,79,83,87,92,96,139,143,147,152,157,162,244,270,274],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[68,72,76,80,84,88,93,97,140,144,148,153,158,163,245,271,275],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Detecting</b>":[69],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">G</b>":[73],"rounding":[74],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Multi</b>":[77],"-":[78],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">M</b>":[81,85,89,145,149],"odal":[82],"edia":[86],"anipulation":[90,150],"(":[91,161],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">DGM<sup>4</sup></b>":[94,98],").":[95],"aims":[99],"to":[100,166,210,284],"not":[101],"detect":[103],"authenticity":[105],"of":[106,119,269],"but":[109],"also":[110,282],"ground":[111],"manipulated":[113],"content,":[114],"which":[115],"requires":[116],"deeper":[117],"media":[121,290],"manipulation.":[122],"To":[123,219],"support":[124],"large-scale":[126],"investigation,":[127],"construct":[129,239],"first":[131],"DGM4":[132],"dataset.":[133],"Moreover,":[134],"propose":[136],"novel":[138],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Hier</b>":[141],"Archical":[142],"ulti-modal":[146],"r":[151],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">E</b>":[154],"asoning":[155],"t":[156],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">R</b>":[159],"ansformer":[160],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">HAMMER</b>":[164,272],")":[165],"fully":[167],"capture":[168],"fine-grained":[170,222],"interaction":[171],"between":[172,181],"HAMMER":[175],"performs":[176],"1)":[177],"manipulation-aware":[178],"contrastive":[179,223],"learning":[180,224],"two":[182],"uni-modal":[183],"encoders":[184],"as":[185,196],"shallow":[186,209],"manipulation":[187,198,201],"reasoning,":[188],"2)":[190],"modality-aware":[191],"cross-attention":[192],"by":[193],"aggregator":[195],"deep":[197,211],"reasoning.":[199],"Dedicated":[200],"grounding":[204],"heads":[205],"integrated":[207],"from":[208],"levels":[212],"interacted":[216],"information.":[218],"exploit":[220],"more":[221,241],"cross-modal":[226],"semantic":[227],"alignment,":[228],"further":[230],"integrate":[231],"Manipulation-Aware":[232],"Contrastive":[233],"Loss":[234],"with":[235],"Local":[236],"View":[237],"advanced":[242],"model":[243],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">HAMMER++</b>":[246,276],"Finally,":[247],"build":[249],"an":[250],"extensive":[251],"benchmark":[252],"set":[254],"up":[255],"rigorous":[256],"evaluation":[257],"metrics":[258],"problem.":[263],"Comprehensive":[264],"experiments":[265],"demonstrate":[266],"superiority":[268],";":[277],"several":[278],"valuable":[279],"observations":[280],"revealed":[283],"facilitate":[285],"future":[286],"manipulation..":[291]},"counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":24},{"year":2024,"cited_by_count":6}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}