{"id":"https://openalex.org/W4372260514","doi":"https://doi.org/10.1109/icassp49357.2023.10095247","title":"Hearing and Seeing Abnormality: Self-Supervised Audio-Visual Mutual Learning for Deepfake Detection","display_name":"Hearing and Seeing Abnormality: Self-Supervised Audio-Visual Mutual Learning for Deepfake Detection","publication_year":2023,"publication_date":"2023-05-05","ids":{"openalex":"https://openalex.org/W4372260514","doi":"https://doi.org/10.1109/icassp49357.2023.10095247"},"language":"en","primary_location":{"id":"doi:10.1109/icassp49357.2023.10095247","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp49357.2023.10095247","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042042015","display_name":"Chang-Sung Sung","orcid":null},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]},{"id":"https://openalex.org/I4210086894","display_name":"Research Center for Information Technology Innovation, Academia Sinica","ror":"https://ror.org/000zgvm20","country_code":"TW","type":"facility","lineage":["https://openalex.org/I4210086894","https://openalex.org/I84653119"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Chang-Sung Sung","raw_affiliation_strings":["National Taiwan University","Research Center for Information Technology Innovation, Academia Sinica"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Taiwan University","institution_ids":["https://openalex.org/I16733864"]},{"raw_affiliation_string":"Research Center for Information Technology Innovation, Academia Sinica","institution_ids":["https://openalex.org/I4210086894"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010007692","display_name":"Jun-Cheng Chen","orcid":"https://orcid.org/0000-0002-0209-8932"},"institutions":[{"id":"https://openalex.org/I4210086894","display_name":"Research Center for Information Technology Innovation, Academia Sinica","ror":"https://ror.org/000zgvm20","country_code":"TW","type":"facility","lineage":["https://openalex.org/I4210086894","https://openalex.org/I84653119"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Jun-Cheng Chen","raw_affiliation_strings":["Academia Sinica,Research Center for Information Technology Innovation","Research Center for Information Technology Innovation, Academia Sinica"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Academia Sinica,Research Center for Information Technology Innovation","institution_ids":["https://openalex.org/I4210086894"]},{"raw_affiliation_string":"Research Center for Information Technology Innovation, Academia Sinica","institution_ids":["https://openalex.org/I4210086894"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5072890368","display_name":"Chu\u2010Song Chen","orcid":"https://orcid.org/0000-0002-2959-2471"},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Chu-Song Chen","raw_affiliation_strings":["National Taiwan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Taiwan University","institution_ids":["https://openalex.org/I16733864"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12357","display_name":"Digital Media Forensic Detection","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12357","display_name":"Digital Media Forensic Detection","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9954000115394592,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8568516969680786},{"id":"https://openalex.org/keywords/overfitting","display_name":"Overfitting","score":0.793590784072876},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6344112753868103},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.5702347755432129},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.505443811416626},{"id":"https://openalex.org/keywords/labeled-data","display_name":"Labeled data","score":0.49150025844573975},{"id":"https://openalex.org/keywords/feature-learning","display_name":"Feature learning","score":0.46540313959121704},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.42623692750930786},{"id":"https://openalex.org/keywords/abnormality","display_name":"Abnormality","score":0.4163036644458771},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.41380053758621216},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.4087424874305725},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.39897552132606506},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.36634331941604614},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.2418929934501648}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8568516969680786},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.793590784072876},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6344112753868103},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.5702347755432129},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.505443811416626},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.49150025844573975},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.46540313959121704},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.42623692750930786},{"id":"https://openalex.org/C50965678","wikidata":"https://www.wikidata.org/wiki/Q2724302","display_name":"Abnormality","level":2,"score":0.4163036644458771},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.41380053758621216},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.4087424874305725},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39897552132606506},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.36634331941604614},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.2418929934501648},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp49357.2023.10095247","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp49357.2023.10095247","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.8100000023841858}],"awards":[],"funders":[{"id":"https://openalex.org/F4320331164","display_name":"National Science and Technology Council","ror":"https://ror.org/00wnb9798"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W2152790380","https://openalex.org/W2604379605","https://openalex.org/W2842511635","https://openalex.org/W2945262873","https://openalex.org/W2959409678","https://openalex.org/W2982058372","https://openalex.org/W3005680577","https://openalex.org/W3017837134","https://openalex.org/W3034196597","https://openalex.org/W3034301684","https://openalex.org/W3034900344","https://openalex.org/W3035524453","https://openalex.org/W3093077034","https://openalex.org/W3100859887","https://openalex.org/W3108281670","https://openalex.org/W3174508664","https://openalex.org/W3196204467","https://openalex.org/W3197123936","https://openalex.org/W3197831948","https://openalex.org/W3205798727","https://openalex.org/W3207531490","https://openalex.org/W4214691743","https://openalex.org/W4287758545","https://openalex.org/W4297808394","https://openalex.org/W4312388562","https://openalex.org/W6682948231","https://openalex.org/W6734491695","https://openalex.org/W6735927292","https://openalex.org/W6762480454","https://openalex.org/W6774314701","https://openalex.org/W6779977557","https://openalex.org/W6782358524","https://openalex.org/W6784810453","https://openalex.org/W6810138774"],"related_works":["https://openalex.org/W4362597605","https://openalex.org/W1574414179","https://openalex.org/W3204418343","https://openalex.org/W4292388283","https://openalex.org/W1560624709","https://openalex.org/W3214142563","https://openalex.org/W3166286441","https://openalex.org/W4221162086","https://openalex.org/W3111760155","https://openalex.org/W2799180539"],"abstract_inverted_index":{"The":[0,102],"recent":[1,19],"development":[2,71],"of":[3,25,61,72,105,261],"deepfakes":[4,93],"has":[5,113,209],"resulted":[6],"in":[7,127,147],"serious":[8],"threats":[9],"to":[10,56,109,158,192,202,228,264],"society,":[11],"such":[12],"as":[13,46],"spreading":[14],"misinformation,":[15],"defamation,":[16],"etc.":[17],"Although":[18],"deepfake":[20],"detection":[21,97],"methods":[22],"are":[23],"capable":[24],"achieving":[26],"satisfactory":[27],"results":[28],"for":[29,36,64,87,91,99],"seen":[30],"forgeries,":[31],"the":[32,48,54,69,119,122,125,133,136,140,153,166,171,194,207,213,217,224,234,241,259],"performance":[33],"drops":[34],"significantly":[35],"unseen":[37,100,251,255],"ones.":[38,101],"With":[39],"proper":[40],"supervised":[41],"pretraining":[42],"on":[43,165,245,250],"auxiliary":[44],"tasks":[45,66],"prior,":[47],"situation":[49],"can":[50],"be":[51],"improved,":[52],"but":[53],"requirement":[55],"collect":[57],"a":[58,73,115,128,148,161],"large":[59],"number":[60],"additional":[62],"annotations":[63],"these":[65],"may":[67],"restrict":[68],"further":[70],"generalized":[74],"deep-fake":[75],"detector.":[76],"To":[77,222],"address":[78],"this":[79],"issue,":[80],"we":[81,179,232],"propose":[82],"an":[83],"Audio-Visual":[84],"Temporal":[85],"Synchronization":[86],"Deepfake":[88],"Detection":[89],"framework":[90,107],"detecting":[92],"that":[94],"maintains":[95],"reasonable":[96],"capabilities":[98],"primary":[103],"objective":[104],"our":[106,262],"is":[108,145,200],"determine":[110,203],"whether":[111,204],"there":[112],"been":[114,210],"forgery":[116,252],"by":[117,151],"evaluating":[118],"consistency":[120],"between":[121,135,170],"sound":[123],"and":[124,173,184,238,254],"faces":[126],"video":[129,208],"clip,":[130],"together":[131],"with":[132,189],"relationship":[134,169],"two":[137],"features.":[138],"First,":[139],"spatiotemporal":[141],"feature":[142,219,235],"extraction":[143,220,236],"network":[144,199,244],"pretrained":[146,218],"self-supervised":[149],"manner":[150],"exploiting":[152],"audio-visual":[154],"temporal":[155,167,197],"synchronization":[156,168],"task":[157],"build":[159],"up":[160],"rich":[162],"representation":[163],"based":[164],"audio":[172],"its":[174],"corresponding":[175],"video.":[176],"For":[177],"pretraining,":[178],"use":[180],"only":[181,239],"real":[182],"data":[183],"carefully":[185],"selected":[186],"negative":[187],"samples":[188],"contrastive":[190],"loss":[191],"train":[193],"model.":[195],"A":[196],"classifier":[198,243],"used":[201],"or":[205],"not":[206],"manipulated":[211],"using":[212],"representations":[214],"obtained":[215],"from":[216,226],"networks.":[221],"prevent":[223],"model":[225],"overfitting":[227],"certain":[229],"manipulation-specific":[230],"artifacts,":[231],"froze":[233],"networks":[237],"trained":[240],"final":[242],"forged":[246],"data.":[247],"Extensive":[248],"experiments":[249],"categories":[253],"datasets":[256],"have":[257],"shown":[258],"effectiveness":[260],"method":[263],"achieve":[265],"state-of-the-art":[266],"results.":[267]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}