{"id":"https://openalex.org/W4385807453","doi":"https://doi.org/10.21437/interspeech.2023-2193","title":"Whisper-AT: Noise-Robust Automatic Speech Recognizers are Also Strong General Audio Event Taggers","display_name":"Whisper-AT: Noise-Robust Automatic Speech Recognizers are Also Strong General Audio Event Taggers","publication_year":2023,"publication_date":"2023-08-14","ids":{"openalex":"https://openalex.org/W4385807453","doi":"https://doi.org/10.21437/interspeech.2023-2193"},"language":"en","primary_location":{"id":"doi:10.21437/interspeech.2023-2193","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2023-2193","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"INTERSPEECH 2023","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067285379","display_name":"Yuan Gong","orcid":"https://orcid.org/0000-0002-4537-0078"},"institutions":[{"id":"https://openalex.org/I4405258931","display_name":"MIT Computer Science and Artificial Intelligence Laboratory","ror":"https://ror.org/01e85r662","country_code":"US","type":"facility","lineage":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuan Gong","raw_affiliation_strings":["MIT CSAIL, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT CSAIL, USA","institution_ids":["https://openalex.org/I4405258931"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035187454","display_name":"Sameer Khurana","orcid":"https://orcid.org/0000-0002-3182-1085"},"institutions":[{"id":"https://openalex.org/I4405258931","display_name":"MIT Computer Science and Artificial Intelligence Laboratory","ror":"https://ror.org/01e85r662","country_code":"US","type":"facility","lineage":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sameer Khurana","raw_affiliation_strings":["MIT CSAIL, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT CSAIL, USA","institution_ids":["https://openalex.org/I4405258931"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020676344","display_name":"Leonid Karlinsky","orcid":"https://orcid.org/0000-0003-2524-2068"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Leonid Karlinsky","raw_affiliation_strings":["MIT-IBM Watson AI Lab, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT-IBM Watson AI Lab, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112758056","display_name":"James Glass","orcid":"https://orcid.org/0000-0002-3097-360X"},"institutions":[{"id":"https://openalex.org/I4405258931","display_name":"MIT Computer Science and Artificial Intelligence Laboratory","ror":"https://ror.org/01e85r662","country_code":"US","type":"facility","lineage":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James Glass","raw_affiliation_strings":["MIT CSAIL, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT CSAIL, USA","institution_ids":["https://openalex.org/I4405258931"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":60,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2798","last_page":"2802"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9905999898910522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9905999898910522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9900000095367432,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9747999906539917,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8059381246566772},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.7941861152648926},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.5725171566009521},{"id":"https://openalex.org/keywords/event","display_name":"Event (particle physics)","score":0.5655633211135864},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4084954261779785},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.38959190249443054}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8059381246566772},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.7941861152648926},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.5725171566009521},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.5655633211135864},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4084954261779785},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38959190249443054},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.21437/interspeech.2023-2193","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2023-2193","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"INTERSPEECH 2023","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W1494198834","https://openalex.org/W1522301498","https://openalex.org/W2062164080","https://openalex.org/W2129674987","https://openalex.org/W2584667682","https://openalex.org/W2593116425","https://openalex.org/W2885724687","https://openalex.org/W2962909949","https://openalex.org/W2995181338","https://openalex.org/W3036601975","https://openalex.org/W3094550259","https://openalex.org/W3095738461","https://openalex.org/W3174906557","https://openalex.org/W3196974791","https://openalex.org/W3198771897","https://openalex.org/W3205743929","https://openalex.org/W3206996142","https://openalex.org/W3209059054","https://openalex.org/W4221140371","https://openalex.org/W4221150524","https://openalex.org/W4224932123","https://openalex.org/W4285483774","https://openalex.org/W4297841853","https://openalex.org/W4311000453","https://openalex.org/W4385245566","https://openalex.org/W4398958419"],"related_works":["https://openalex.org/W2789919619","https://openalex.org/W2293457016","https://openalex.org/W3169305685","https://openalex.org/W2351428524","https://openalex.org/W2368779261","https://openalex.org/W1551406738","https://openalex.org/W2610387714","https://openalex.org/W2369308426","https://openalex.org/W1569841287","https://openalex.org/W1512718085"],"abstract_inverted_index":{"In":[0],"this":[1,69],"paper,":[2],"we":[3,71],"focus":[4],"on":[5,65,95],"Whisper":[6,34,61],"[1],":[7],"a":[8,16,73,90,114],"recent":[9],"automatic":[10],"speech":[11,21,63,78],"recognition":[12,79],"model":[13,80,94],"trained":[14],"with":[15],"massive":[17],"680k":[18],"hour":[19],"labeled":[20],"corpus":[22],"recorded":[23],"in":[24,108,113],"diverse":[25],"conditions.We":[26],"first":[27],"show":[28],"an":[29],"interesting":[30],"finding":[31],"that":[32,60],"while":[33],"is":[35,47,52],"very":[36],"robust":[37],"against":[38],"real-world":[39],"background":[40],"sounds":[41],"(e.g.,":[42],"music),":[43],"its":[44],"audio":[45,75,92,106],"representation":[46],"actually":[48],"not":[49],"noise-invariant,":[50],"but":[51],"instead":[53],"highly":[54],"correlated":[55],"to":[56,110],"non-speech":[57],"sounds,":[58],"indicating":[59],"recognizes":[62],"conditioned":[64],"the":[66,84],"noise":[67],"type.With":[68],"finding,":[70],"build":[72],"unified":[74],"tagging":[76,93],"and":[77,88],"Whisper-AT":[81,103],"by":[82],"freezing":[83],"backbone":[85],"of":[86,97],"Whisper,":[87],"training":[89],"lightweight":[91],"top":[96],"it.With":[98],"<1%":[99],"extra":[100],"computational":[101],"cost,":[102],"can":[104],"recognize":[105],"events,":[107],"addition":[109],"spoken":[111],"text,":[112],"single":[115],"forward":[116],"pass.":[117]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":29},{"year":2024,"cited_by_count":20},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}