{"id":"https://openalex.org/W4415540621","doi":"https://doi.org/10.1145/3746027.3755639","title":"AV-RISE: Hierarchical Cross-Modal Denoising for Learning Robust Audio-Visual Speech Representation","display_name":"AV-RISE: Hierarchical Cross-Modal Denoising for Learning Robust Audio-Visual Speech Representation","publication_year":2025,"publication_date":"2025-10-25","ids":{"openalex":"https://openalex.org/W4415540621","doi":"https://doi.org/10.1145/3746027.3755639"},"language":null,"primary_location":{"id":"doi:10.1145/3746027.3755639","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3746027.3755639","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102575151","display_name":"Zhishuo Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I24185976","display_name":"Sichuan University","ror":"https://ror.org/011ashp19","country_code":"CN","type":"education","lineage":["https://openalex.org/I24185976"]},{"id":"https://openalex.org/I4210125143","display_name":"Chengdu University","ror":"https://ror.org/034z67559","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210125143"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Zhishuo Zhao","raw_affiliation_strings":["College of Computer Science, Sichuan University, Chengdu, China"],"affiliations":[{"raw_affiliation_string":"College of Computer Science, Sichuan University, Chengdu, China","institution_ids":["https://openalex.org/I4210125143","https://openalex.org/I24185976"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066617875","display_name":"Yi Lin","orcid":"https://orcid.org/0000-0002-7194-5023"},"institutions":[{"id":"https://openalex.org/I24185976","display_name":"Sichuan University","ror":"https://ror.org/011ashp19","country_code":"CN","type":"education","lineage":["https://openalex.org/I24185976"]},{"id":"https://openalex.org/I4210125143","display_name":"Chengdu University","ror":"https://ror.org/034z67559","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210125143"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yi Lin","raw_affiliation_strings":["College of Computer Science, Sichuan University, Chengdu, China"],"affiliations":[{"raw_affiliation_string":"College of Computer Science, Sichuan University, Chengdu, China","institution_ids":["https://openalex.org/I4210125143","https://openalex.org/I24185976"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057805812","display_name":"Dongyue Guo","orcid":"https://orcid.org/0000-0003-0393-5197"},"institutions":[{"id":"https://openalex.org/I24185976","display_name":"Sichuan University","ror":"https://ror.org/011ashp19","country_code":"CN","type":"education","lineage":["https://openalex.org/I24185976"]},{"id":"https://openalex.org/I4210125143","display_name":"Chengdu University","ror":"https://ror.org/034z67559","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210125143"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongyue Guo","raw_affiliation_strings":["College of Computer Science, Sichuan University, Chengdu, China"],"affiliations":[{"raw_affiliation_string":"College of Computer Science, Sichuan University, Chengdu, China","institution_ids":["https://openalex.org/I4210125143","https://openalex.org/I24185976"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5115425497","display_name":"Junyu Fan","orcid":"https://orcid.org/0009-0009-9848-4261"},"institutions":[{"id":"https://openalex.org/I24185976","display_name":"Sichuan University","ror":"https://ror.org/011ashp19","country_code":"CN","type":"education","lineage":["https://openalex.org/I24185976"]},{"id":"https://openalex.org/I4210125143","display_name":"Chengdu University","ror":"https://ror.org/034z67559","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210125143"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junyu Fan","raw_affiliation_strings":["College of Computer Science, Sichuan University, Chengdu, China"],"affiliations":[{"raw_affiliation_string":"College of Computer Science, Sichuan University, Chengdu, China","institution_ids":["https://openalex.org/I4210125143","https://openalex.org/I24185976"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5102575151"],"corresponding_institution_ids":["https://openalex.org/I24185976","https://openalex.org/I4210125143"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.3937248,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2054","last_page":"2063"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4973999857902527},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.47290000319480896},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4713999927043915},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.46399998664855957},{"id":"https://openalex.org/keywords/noise-reduction","display_name":"Noise reduction","score":0.3955000042915344},{"id":"https://openalex.org/keywords/fusion","display_name":"Fusion","score":0.3921000063419342},{"id":"https://openalex.org/keywords/discriminative-model","display_name":"Discriminative model","score":0.39100000262260437},{"id":"https://openalex.org/keywords/fusion-mechanism","display_name":"Fusion mechanism","score":0.385699987411499}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.722599983215332},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6302000284194946},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.609000027179718},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4973999857902527},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.47290000319480896},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4713999927043915},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.46399998664855957},{"id":"https://openalex.org/C163294075","wikidata":"https://www.wikidata.org/wiki/Q581861","display_name":"Noise reduction","level":2,"score":0.3955000042915344},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.3921000063419342},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.39100000262260437},{"id":"https://openalex.org/C173414695","wikidata":"https://www.wikidata.org/wiki/Q5510276","display_name":"Fusion mechanism","level":4,"score":0.385699987411499},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.3815999925136566},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.3734000027179718},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.3716000020503998},{"id":"https://openalex.org/C2776182073","wikidata":"https://www.wikidata.org/wiki/Q7575395","display_name":"Speech enhancement","level":3,"score":0.33239999413490295},{"id":"https://openalex.org/C100675267","wikidata":"https://www.wikidata.org/wiki/Q1371624","display_name":"Background noise","level":2,"score":0.322299987077713},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.303600013256073},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2824999988079071},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.2777000069618225},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.26649999618530273},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.25119999051094055}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3746027.3755639","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3746027.3755639","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W1536680647","https://openalex.org/W1995562189","https://openalex.org/W2076462394","https://openalex.org/W2121486117","https://openalex.org/W2134867751","https://openalex.org/W2168961642","https://openalex.org/W2551572271","https://openalex.org/W2735399860","https://openalex.org/W2884585870","https://openalex.org/W3016096216","https://openalex.org/W3162028189","https://openalex.org/W3171007011","https://openalex.org/W4285149123","https://openalex.org/W4312910992","https://openalex.org/W4362610547","https://openalex.org/W4372346152"],"related_works":[],"abstract_inverted_index":{"Audio-visual":[0],"speech":[1,10,76,92],"recognition":[2],"(AVSR)":[3],"leverages":[4],"complementary":[5,148],"visual":[6,109],"cues":[7],"to":[8,45,84,88,104,138,181],"improve":[9],"recognition.":[11],"However,":[12],"in":[13,34,50,55,68],"real-world":[14],"scenarios,":[15,26],"both":[16],"modalities":[17],"may":[18,43],"suffer":[19],"from":[20,94],"noise":[21,191],"or":[22],"occlusion.":[23],"In":[24,127],"such":[25],"most":[27],"existing":[28],"fusion":[29,52,100,175],"strategies":[30],"overlook":[31],"the":[32,51,113,124,128,145,154,172],"variation":[33],"modality-specific":[35],"quality":[36],"under":[37,167],"different":[38],"degradation":[39],"conditions.":[40],"This":[41],"limitation":[42],"lead":[44],"dominance":[46],"of":[47,123,147],"corrupted":[48,95],"modality":[49],"process,":[53],"resulting":[54],"worse":[56],"AVSR":[57,164],"performance":[58],"than":[59],"unimodal":[60],"systems,":[61],"termed":[62],"as":[63],"Corrupted":[64],"Modality":[65],"Bias":[66],"(CMB)":[67],"this":[69],"work.":[70],"To":[71],"address":[72],"this,":[73],"a":[74],"self-supervised":[75],"representation":[77],"learning":[78],"framework,":[79],"called":[80],"AV-RISE,":[81],"is":[82,102],"proposed":[83],"employ":[85],"teacher-student":[86],"self-distillation":[87],"robustly":[89],"reconstruct":[90],"clean":[91,189],"representations":[93,110,180],"audio-visual":[96],"inputs.":[97],"A":[98],"hierarchical":[99,173],"mechanism":[101],"designed":[103],"progressively":[105],"refine":[106],"audio":[107],"and":[108,115,133,156,190],"by":[111,184],"integrating":[112],"Suppression":[114],"Enhancement":[116],"Interaction":[117],"(SEI)":[118],"module":[119],"into":[120],"each":[121],"layer":[122],"pre-trained":[125],"encoder.":[126],"SEI":[129],"module,":[130],"cross-modal":[131],"suppression":[132],"modality-oriented":[134],"enhancement":[135],"are":[136],"performed":[137],"mitigate":[139,182],"noise-induced":[140],"feature":[141,186],"inconsistencies,":[142],"which":[143],"strengthens":[144],"modeling":[146],"semantic":[149,179],"representations.":[150],"Extensive":[151],"experiments":[152],"on":[153],"LRS2":[155],"LRS3":[157],"datasets":[158],"demonstrate":[159],"that":[160],"AV-RISE":[161],"outperforms":[162],"SOTA":[163],"models,":[165],"especially":[166],"extreme":[168],"degradation.":[169],"Most":[170],"importantly,":[171],"SEI-based":[174],"effectively":[176],"enhances":[177],"reliable":[178],"CMB,":[183],"evaluating":[185],"similarities":[187],"between":[188],"samples.":[192]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-25T00:00:00"}