{"id":"https://openalex.org/W4401597750","doi":"https://doi.org/10.1109/icasspw62465.2024.10626117","title":"Attention or Convolution: Transformer Encoders in Audio Language Models for Inference Efficiency","display_name":"Attention or Convolution: Transformer Encoders in Audio Language Models for Inference Efficiency","publication_year":2024,"publication_date":"2024-04-14","ids":{"openalex":"https://openalex.org/W4401597750","doi":"https://doi.org/10.1109/icasspw62465.2024.10626117"},"language":"en","primary_location":{"id":"doi:10.1109/icasspw62465.2024.10626117","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/icasspw62465.2024.10626117","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Acoustics, Speech, and Signal Processing Workshops (ICASSPW)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010937599","display_name":"Sungho Jeon","orcid":"https://orcid.org/0000-0002-0417-6906"},"institutions":[{"id":"https://openalex.org/I71030271","display_name":"Heidelberg Institute for Theoretical Studies","ror":"https://ror.org/01f7bcy98","country_code":"DE","type":"facility","lineage":["https://openalex.org/I71030271"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sungho Jeon","raw_affiliation_strings":["Heidelberg Institute of Theoretical Studies"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Heidelberg Institute of Theoretical Studies","institution_ids":["https://openalex.org/I71030271"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087421101","display_name":"Ching-Feng Yeh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ching-Feng Yeh","raw_affiliation_strings":["Meta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071606441","display_name":"Hakan Inan","orcid":"https://orcid.org/0000-0003-3505-8838"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hakan Inan","raw_affiliation_strings":["Meta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051950818","display_name":"Wei-Ning Hsu","orcid":"https://orcid.org/0000-0001-5546-5217"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei-Ning Hsu","raw_affiliation_strings":["Meta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035660188","display_name":"Rashi Rungta","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rashi Rungta","raw_affiliation_strings":["Meta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067997502","display_name":"Yashar Mehdad","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yashar Mehdad","raw_affiliation_strings":["Meta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111888905","display_name":"Daniel M. Bikel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Daniel Bikel","raw_affiliation_strings":["Meta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.12329176,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"33","issue":null,"first_page":"555","last_page":"559"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9864000082015991,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9864000082015991,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9797000288963318,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9503999948501587,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.708633303642273},{"id":"https://openalex.org/keywords/convolution","display_name":"Convolution (computer science)","score":0.6082325577735901},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5887129306793213},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5718740224838257},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5022354125976562},{"id":"https://openalex.org/keywords/overlap\u2013add-method","display_name":"Overlap\u2013add method","score":0.4601043462753296},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4414336085319519},{"id":"https://openalex.org/keywords/electronic-engineering","display_name":"Electronic engineering","score":0.32347360253334045},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.24050867557525635},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.16425853967666626},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.1332242488861084},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12126216292381287},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1109844446182251},{"id":"https://openalex.org/keywords/fourier-transform","display_name":"Fourier transform","score":0.08968859910964966}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.708633303642273},{"id":"https://openalex.org/C45347329","wikidata":"https://www.wikidata.org/wiki/Q5166604","display_name":"Convolution (computer science)","level":3,"score":0.6082325577735901},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5887129306793213},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5718740224838257},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5022354125976562},{"id":"https://openalex.org/C181002996","wikidata":"https://www.wikidata.org/wiki/Q1611641","display_name":"Overlap\u2013add method","level":5,"score":0.4601043462753296},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4414336085319519},{"id":"https://openalex.org/C24326235","wikidata":"https://www.wikidata.org/wiki/Q126095","display_name":"Electronic engineering","level":1,"score":0.32347360253334045},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.24050867557525635},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.16425853967666626},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.1332242488861084},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12126216292381287},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1109844446182251},{"id":"https://openalex.org/C102519508","wikidata":"https://www.wikidata.org/wiki/Q6520159","display_name":"Fourier transform","level":2,"score":0.08968859910964966},{"id":"https://openalex.org/C203024314","wikidata":"https://www.wikidata.org/wiki/Q1365258","display_name":"Fourier analysis","level":3,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C76563020","wikidata":"https://www.wikidata.org/wiki/Q4817582","display_name":"Fractional Fourier transform","level":4,"score":0.0},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icasspw62465.2024.10626117","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/icasspw62465.2024.10626117","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Acoustics, Speech, and Signal Processing Workshops (ICASSPW)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W2973049979","https://openalex.org/W3024366773","https://openalex.org/W3036601975","https://openalex.org/W3081168214","https://openalex.org/W3097777922","https://openalex.org/W3197580070","https://openalex.org/W3200129129","https://openalex.org/W3206996142","https://openalex.org/W3209059054","https://openalex.org/W3209984917","https://openalex.org/W4281708609","https://openalex.org/W4281779489","https://openalex.org/W4295262505","https://openalex.org/W4297841520","https://openalex.org/W4297841794","https://openalex.org/W4313484417","https://openalex.org/W4323654151","https://openalex.org/W6693397755","https://openalex.org/W6761628794","https://openalex.org/W6780218876","https://openalex.org/W6784333009","https://openalex.org/W6784614252","https://openalex.org/W6838276489","https://openalex.org/W6838683425","https://openalex.org/W6848709529"],"related_works":["https://openalex.org/W4372260258","https://openalex.org/W2267589039","https://openalex.org/W2369791303","https://openalex.org/W4206178588","https://openalex.org/W3094491777","https://openalex.org/W3214715529","https://openalex.org/W2133280289","https://openalex.org/W2360069155","https://openalex.org/W4287635093","https://openalex.org/W2759540840"],"abstract_inverted_index":{"In":[0],"this":[1,84],"paper,":[2],"we":[3,72,132],"show":[4,48],"that":[5,49,71,83,105,131],"a":[6,91,97],"simple":[7],"audio":[8,62,140],"language":[9,63,141,142],"model":[10],"can":[11,73],"achieve":[12,37,74],"comparable":[13,75],"inference":[14,146],"efficiency":[15,60,76],"to":[16,100,116,136,143],"more":[17],"complicated":[18],"pre-trained":[19],"models":[20,64],"with":[21,33,43,77,90],"speech":[22,26,52,118],"transformer":[23],"encoders.":[24],"These":[25],"transformers":[27,53,119],"rely":[28],"on":[29,41],"mixing":[30,120],"convolutional":[31],"modules":[32,114],"self-attention":[34,79,126],"modules.":[35,127],"They":[36],"state-of-":[38],"the-art":[39],"performance":[40],"ASR":[42],"top":[44],"efficiency.":[45,102,147],"We":[46,81,103],"first":[47],"employing":[50],"these":[51],"as":[54,65],"an":[55],"encoder":[56],"significantly":[57],"improves":[58],"the":[59,109,124,137],"of":[61,96,139],"well.":[66],"However,":[67],"our":[68],"study":[69,129],"shows":[70],"advanced":[78],"solely.":[80],"demonstrate":[82],"simpler":[85],"approach":[86],"is":[87],"particularly":[88],"beneficial":[89],"low-bit":[92],"weight":[93],"quantization":[94],"technique":[95],"neural":[98],"network":[99],"improve":[101,144],"hypothesize":[104],"it":[106],"prevents":[107],"propagating":[108],"errors":[110],"between":[111],"different":[112],"quantized":[113,121,125],"compared":[115],"recent":[117],"convolution":[122],"and":[123],"Our":[128],"suggests":[130],"could":[133],"pay":[134],"attention":[135],"architecture":[138],"their":[145]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}