{"id":"https://openalex.org/W4401597702","doi":"https://doi.org/10.1109/icasspw62465.2024.10627466","title":"Multimodal Attention Merging for Improved Speech Recognition and Audio Event Classification","display_name":"Multimodal Attention Merging for Improved Speech Recognition and Audio Event Classification","publication_year":2024,"publication_date":"2024-04-14","ids":{"openalex":"https://openalex.org/W4401597702","doi":"https://doi.org/10.1109/icasspw62465.2024.10627466"},"language":"en","primary_location":{"id":"doi:10.1109/icasspw62465.2024.10627466","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icasspw62465.2024.10627466","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Acoustics, Speech, and Signal Processing Workshops (ICASSPW)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013573557","display_name":"Anirudh S. Sundar","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anirudh S. Sundar","raw_affiliation_strings":["Georgia Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020376803","display_name":"Chao-Han Huck Yang","orcid":"https://orcid.org/0000-0003-2879-8811"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chao-Han Huck Yang","raw_affiliation_strings":["Amazon Alexa AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104260047","display_name":"David M. Chan","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David M. Chan","raw_affiliation_strings":["Amazon Alexa AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102095407","display_name":"Shalini Ghosh","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shalini Ghosh","raw_affiliation_strings":["Amazon Alexa AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060593241","display_name":"Venkatesh Ravichandran","orcid":"https://orcid.org/0009-0001-7214-2919"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Venkatesh Ravichandran","raw_affiliation_strings":["Amazon Alexa AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5005758272","display_name":"Phani Sankar Nidadavolu","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Phani Sankar Nidadavolu","raw_affiliation_strings":["Amazon Alexa AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI","institution_ids":["https://openalex.org/I1311688040"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"655","last_page":"659"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.982699990272522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.982699990272522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9739000201225281,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9488999843597412,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8070623874664307},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.7037342190742493},{"id":"https://openalex.org/keywords/event","display_name":"Event (particle physics)","score":0.609420657157898},{"id":"https://openalex.org/keywords/audio-mining","display_name":"Audio mining","score":0.5030028223991394},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.41722673177719116},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3426918387413025},{"id":"https://openalex.org/keywords/speech-processing","display_name":"Speech processing","score":0.31534484028816223},{"id":"https://openalex.org/keywords/acoustic-model","display_name":"Acoustic model","score":0.24930748343467712}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8070623874664307},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.7037342190742493},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.609420657157898},{"id":"https://openalex.org/C157968479","wikidata":"https://www.wikidata.org/wiki/Q3079876","display_name":"Audio mining","level":4,"score":0.5030028223991394},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41722673177719116},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3426918387413025},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.31534484028816223},{"id":"https://openalex.org/C155635449","wikidata":"https://www.wikidata.org/wiki/Q4674699","display_name":"Acoustic model","level":3,"score":0.24930748343467712},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icasspw62465.2024.10627466","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icasspw62465.2024.10627466","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Acoustics, Speech, and Signal Processing Workshops (ICASSPW)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":44,"referenced_works":["https://openalex.org/W1494198834","https://openalex.org/W1861492603","https://openalex.org/W1933349210","https://openalex.org/W2052666245","https://openalex.org/W2108598243","https://openalex.org/W2593116425","https://openalex.org/W2896457183","https://openalex.org/W2913190747","https://openalex.org/W2962767983","https://openalex.org/W2998572311","https://openalex.org/W3013325675","https://openalex.org/W3094502228","https://openalex.org/W3134095442","https://openalex.org/W3168867926","https://openalex.org/W3198802556","https://openalex.org/W3209059054","https://openalex.org/W4226037452","https://openalex.org/W4226089239","https://openalex.org/W4226405181","https://openalex.org/W4283794074","https://openalex.org/W4307206164","https://openalex.org/W4310998073","https://openalex.org/W4312048190","https://openalex.org/W4385245566","https://openalex.org/W4385822823","https://openalex.org/W4386566605","https://openalex.org/W4389520072","https://openalex.org/W4390190013","https://openalex.org/W6757817989","https://openalex.org/W6758448228","https://openalex.org/W6759401939","https://openalex.org/W6759579507","https://openalex.org/W6769190946","https://openalex.org/W6771812881","https://openalex.org/W6778883912","https://openalex.org/W6784333009","https://openalex.org/W6796456916","https://openalex.org/W6804122979","https://openalex.org/W6810089409","https://openalex.org/W6810127701","https://openalex.org/W6810599859","https://openalex.org/W6846305912","https://openalex.org/W6847030267","https://openalex.org/W6848208918"],"related_works":["https://openalex.org/W2539919382","https://openalex.org/W22517275","https://openalex.org/W2042419400","https://openalex.org/W2122924390","https://openalex.org/W2064347532","https://openalex.org/W2589081144","https://openalex.org/W2171620991","https://openalex.org/W189256204","https://openalex.org/W2981428355","https://openalex.org/W2131711534"],"abstract_inverted_index":{"Training":[0],"large":[1],"foundation":[2],"models":[3,59],"using":[4],"self-supervised":[5],"objectives":[6],"on":[7,13],"unlabeled":[8],"data,":[9],"followed":[10],"by":[11,31,94,109],"fine-tuning":[12,34,147],"downstream":[14,40],"tasks,":[15],"has":[16],"emerged":[17],"as":[18],"a":[19,76,121,130],"standard":[20],"procedure.":[21],"Unfortunately,":[22],"the":[23,81],"efficacy":[24],"of":[25,58,87,102],"this":[26],"approach":[27,123],"is":[28,116],"often":[29],"constrained":[30],"both":[32],"limited":[33],"compute":[35],"and":[36,65,73,98,139],"scarcity":[37],"in":[38,61,69,129,135,143],"labeled":[39],"data.":[41],"We":[42],"introduce":[43],"Multimodal":[44],"Attention":[45],"Merging":[46],"(MAM),":[47],"an":[48,88,103],"attempt":[49],"that":[50],"facilitates":[51],"direct":[52],"knowledge":[53],"transfer":[54],"from":[55],"attention":[56,126],"matrices":[57],"rooted":[60],"high-resource":[62],"modalities,":[63],"text":[64],"images,":[66],"to":[67,96,124,146],"those":[68],"resource-constrained":[70],"domains,":[71],"speech":[72],"audio,":[74],"employing":[75],"zero-shot":[77],"paradigm.":[78],"MAM":[79],"reduces":[80],"relative":[82,99,133,141],"Word":[83],"Error":[84],"Rate":[85],"(WER)":[86],"Automatic":[89],"Speech":[90],"Recognition":[91],"(ASR)":[92],"model":[93,108,149],"up":[95],"6.70%,":[97],"classification":[100],"error":[101],"Audio":[104],"Event":[105],"Classification":[106],"(AEC)":[107],"10.63%.":[110],"In":[111],"cases":[112],"where":[113],"some":[114],"data/compute":[115],"available,":[117],"we":[118],"present":[119],"Learnable-MAM,":[120],"data-driven":[122],"merging":[125],"matrices,":[127],"resulting":[128],"further":[131],"2.90%":[132],"reduction":[134,142],"WER":[136],"for":[137],"ASR":[138],"18.42%":[140],"AEC":[144],"compared":[145],"without":[148],"merging.":[150]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}