{"id":"https://openalex.org/W4402118924","doi":"https://doi.org/10.21437/interspeech.2024-2105","title":"Outlier Reduction with Gated Attention for Improved Post-training Quantization in Large Sequence-to-sequence Speech Foundation Models","display_name":"Outlier Reduction with Gated Attention for Improved Post-training Quantization in Large Sequence-to-sequence Speech Foundation Models","publication_year":2024,"publication_date":"2024-09-01","ids":{"openalex":"https://openalex.org/W4402118924","doi":"https://doi.org/10.21437/interspeech.2024-2105"},"language":"en","primary_location":{"id":"doi:10.21437/interspeech.2024-2105","doi":"https://doi.org/10.21437/interspeech.2024-2105","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2024-2105","pdf_url":null,"source":{"id":"https://openalex.org/S4393918409","display_name":"Interspeech (USB)","issn_l":"2308-457X","issn":["2308-457X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-1","norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2024","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2406.11022","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016617263","display_name":"Dominik Wagner","orcid":"https://orcid.org/0000-0002-3631-4138","observed_orcids":["https://orcid.org/0000-0002-3631-4138"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dominik Wagner","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047469130","display_name":"Ilja Baumann","orcid":"https://orcid.org/0000-0002-1991-3710","observed_orcids":["https://orcid.org/0000-0002-1991-3710"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ilja Baumann","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053498662","display_name":"Korbinian Riedhammer","orcid":"https://orcid.org/0000-0003-3582-2154","observed_orcids":["https://orcid.org/0000-0003-3582-2154"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Korbinian Riedhammer","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5033302750","display_name":"Tobias Bocklet","orcid":"https://orcid.org/0009-0008-7780-8821","observed_orcids":["https://orcid.org/0009-0008-7780-8821"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tobias Bocklet","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.655,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.74485889,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"4623","last_page":"4627"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.5454000234603882,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.5454000234603882,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.4291999936103821,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.0035000001080334187,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/post-training-quantization","display_name":"post-training quantization","score":0.9760000109672546},{"id":"https://openalex.org/keywords/outlier-reduction","display_name":"outlier reduction","score":0.8619999885559082},{"id":"https://openalex.org/keywords/gated-attention","display_name":"gated attention","score":0.8859999775886536},{"id":"https://openalex.org/keywords/whisper","display_name":"Whisper","score":0.5879999995231628},{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":0.8420000076293945},{"id":"https://openalex.org/keywords/speech-foundation-models","display_name":"speech foundation models","score":0.7409999966621399},{"id":"https://openalex.org/keywords/sequence-to-sequence-models","display_name":"sequence-to-sequence models","score":0.8069999814033508},{"id":"https://openalex.org/keywords/8-bit-quantization","display_name":"8-bit quantization","score":0.9190000295639038},{"id":"https://openalex.org/keywords/word-error-rate","display_name":"word error rate","score":0.8399999737739563},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"speech recognition","score":0.703000009059906}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.661614179611206},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.6461362838745117},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.6444460153579712},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.561039388179779},{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.5270528793334961},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4986116886138916},{"id":"https://openalex.org/C2780966255","wikidata":"https://www.wikidata.org/wiki/Q5474306","display_name":"Foundation (evidence)","level":2,"score":0.48494818806648254},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.4464496970176697},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3249831199645996},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.23197713494300842},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12746641039848328},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.21437/interspeech.2024-2105","doi":"https://doi.org/10.21437/interspeech.2024-2105","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2024-2105","pdf_url":null,"source":{"id":"https://openalex.org/S4393918409","display_name":"Interspeech (USB)","issn_l":"2308-457X","issn":["2308-457X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-1","norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2024","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2406.11022","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.11022","pdf_url":"https://arxiv.org/pdf/2406.11022","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:kobv.de-opus4-th-nuernberg:2313","doi":null,"is_oa":false,"landing_page_url":"https://opus4.kobv.de/opus4-ohm/frontdoor/index/index/docId/2313","pdf_url":null,"source":{"id":"https://openalex.org/S4306401730","display_name":"OHMdok (Technische Hochschule N\u00fcrnberg)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210100612","host_organization_name":"Georg Simon Ohm University of Applied Sciences Nuremberg","host_organization_lineage":["https://openalex.org/I4210100612"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conferenceobject"},{"id":"doi:10.48550/arxiv.2406.11022","doi":"https://doi.org/10.48550/arxiv.2406.11022","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2406.11022","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2406.11022","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.11022","pdf_url":"https://arxiv.org/pdf/2406.11022","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"score":0.5799999833106995,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, justice, and strong institutions"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G3992037128","display_name":"HochleistungscomputeCluster","funder_award_id":"440719683","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G6690480278","display_name":null,"funder_award_id":"440719683","funder_id":"https://openalex.org/F4320320873","funder_display_name":"Friedrich-Alexander-Universit\u00e4t Erlangen-N\u00fcrnberg"}],"funders":[{"id":"https://openalex.org/F4320320873","display_name":"Friedrich-Alexander-Universit\u00e4t Erlangen-N\u00fcrnberg","ror":"https://ror.org/00f7hpc57"},{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4402118924.pdf","grobid_xml":"https://content.openalex.org/works/W4402118924.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2381393187","https://openalex.org/W2332779545","https://openalex.org/W3006513224","https://openalex.org/W2358060160","https://openalex.org/W2035483685","https://openalex.org/W1969764885","https://openalex.org/W596947562","https://openalex.org/W2046456988","https://openalex.org/W2357409937","https://openalex.org/W2510582230"],"abstract_inverted_index":{"This":[0],"paper":[1],"explores":[2],"the":[3,13,21,83,87,104],"improvement":[4],"of":[5,23,86],"post-training":[6],"quantization":[7,33],"(PTQ)":[8],"after":[9],"knowledge":[10],"distillation":[11],"in":[12,25,35,82,107],"Whisper":[14],"speech":[15,62],"foundation":[16],"model":[17],"family.":[18],"We":[19,69],"address":[20],"challenge":[22],"outliers":[24,50,72],"weights":[26],"and":[27,38,94],"activation":[28],"tensors,":[29],"known":[30],"to":[31,44,59,100],"impede":[32],"quality":[34],"transformer-based":[36,55],"language":[37],"vision":[39],"models.":[40],"Extending":[41],"this":[42],"observation":[43],"Whisper,":[45],"we":[46],"demonstrate":[47],"that":[48,71],"these":[49],"are":[51,57],"also":[52],"present":[53],"when":[54],"models":[56,102],"trained":[58],"perform":[60],"automatic":[61],"recognition,":[63],"necessitating":[64],"mitigation":[65],"strategies":[66],"for":[67],"PTQ.":[68],"show":[70],"can":[73],"be":[74],"reduced":[75],"by":[76],"a":[77],"recently":[78],"proposed":[79],"gating":[80,105],"mechanism":[81,106],"attention":[84],"blocks":[85],"student":[88,101],"model,":[89],"enabling":[90],"effective":[91],"8-bit":[92],"quantization,":[93],"lower":[95],"word":[96],"error":[97],"rates":[98],"compared":[99],"without":[103],"place.":[108]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-10-12T09:40:04.860316","created_date":"2025-10-10T00:00:00"}