{"id":"https://openalex.org/W4406495601","doi":"https://doi.org/10.1109/slt61566.2024.10832323","title":"An Analysis of Linear Complexity Attention Substitutes With Best-RQ","display_name":"An Analysis of Linear Complexity Attention Substitutes With Best-RQ","publication_year":2024,"publication_date":"2024-12-02","ids":{"openalex":"https://openalex.org/W4406495601","doi":"https://doi.org/10.1109/slt61566.2024.10832323"},"language":"en","primary_location":{"id":"doi:10.1109/slt61566.2024.10832323","is_oa":false,"landing_page_url":"https://doi.org/10.1109/slt61566.2024.10832323","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE Spoken Language Technology Workshop (SLT)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091012924","display_name":"Ryan Whetten","orcid":null},"institutions":[{"id":"https://openalex.org/I4210119991","display_name":"Laboratoire Informatique d'Avignon","ror":"https://ror.org/02n399288","country_code":"FR","type":"facility","lineage":["https://openalex.org/I198415970","https://openalex.org/I4210119991"]}],"countries":["FR"],"is_corresponding":true,"raw_author_name":"Ryan Whetten","raw_affiliation_strings":["Avignon Université,Laboratoire Informatique d’Avignon,France"],"affiliations":[{"raw_affiliation_string":"Avignon Université,Laboratoire Informatique d’Avignon,France","institution_ids":["https://openalex.org/I4210119991"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089505434","display_name":"Titouan Parcollet","orcid":"https://orcid.org/0000-0003-0672-1346"},"institutions":[{"id":"https://openalex.org/I4210117523","display_name":"Samsung (United Kingdom)","ror":"https://ror.org/01w6gjq94","country_code":"GB","type":"company","lineage":["https://openalex.org/I2250650973","https://openalex.org/I4210117523"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Titouan Parcollet","raw_affiliation_strings":["Samsung AI Center Cambridge,United Kingdom"],"affiliations":[{"raw_affiliation_string":"Samsung AI Center Cambridge,United Kingdom","institution_ids":["https://openalex.org/I4210117523"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066586567","display_name":"Adel Moumen","orcid":null},"institutions":[{"id":"https://openalex.org/I4210119991","display_name":"Laboratoire Informatique d'Avignon","ror":"https://ror.org/02n399288","country_code":"FR","type":"facility","lineage":["https://openalex.org/I198415970","https://openalex.org/I4210119991"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Adel Moumen","raw_affiliation_strings":["Avignon Université,Laboratoire Informatique d’Avignon,France"],"affiliations":[{"raw_affiliation_string":"Avignon Université,Laboratoire Informatique d’Avignon,France","institution_ids":["https://openalex.org/I4210119991"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050707731","display_name":"Marco Dinarelli","orcid":null},"institutions":[{"id":"https://openalex.org/I4210104430","display_name":"Laboratoire d'Informatique de Grenoble","ror":"https://ror.org/01c8rcg82","country_code":"FR","type":"facility","lineage":["https://openalex.org/I106785703","https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I4210104430","https://openalex.org/I4210159245","https://openalex.org/I899635006","https://openalex.org/I899635006"]},{"id":"https://openalex.org/I106785703","display_name":"Institut polytechnique de Grenoble","ror":"https://ror.org/05sbt2524","country_code":"FR","type":"education","lineage":["https://openalex.org/I106785703","https://openalex.org/I899635006"]},{"id":"https://openalex.org/I4210101348","display_name":"Centre Inria de l'Universit\u00e9 Grenoble Alpes","ror":"https://ror.org/00n8d6z93","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1326498283","https://openalex.org/I4210101348"]},{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"funder","lineage":["https://openalex.org/I1294671590"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Marco Dinarelli","raw_affiliation_strings":["Univervisté Grenoble Alpes, Inria, CNRS, Grenoble INP, LIG,Grenoble,France,38000"],"affiliations":[{"raw_affiliation_string":"Univervisté Grenoble Alpes, Inria, CNRS, Grenoble INP, LIG,Grenoble,France,38000","institution_ids":["https://openalex.org/I4210101348","https://openalex.org/I1294671590","https://openalex.org/I4210104430","https://openalex.org/I106785703"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081852717","display_name":"Yannick Est\u00e8ve","orcid":"https://orcid.org/0000-0002-3656-8883"},"institutions":[{"id":"https://openalex.org/I4210119991","display_name":"Laboratoire Informatique d'Avignon","ror":"https://ror.org/02n399288","country_code":"FR","type":"facility","lineage":["https://openalex.org/I198415970","https://openalex.org/I4210119991"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Yannick Est\u00e8ve","raw_affiliation_strings":["Avignon Université,Laboratoire Informatique d’Avignon,France"],"affiliations":[{"raw_affiliation_string":"Avignon Université,Laboratoire Informatique d’Avignon,France","institution_ids":["https://openalex.org/I4210119991"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5091012924"],"corresponding_institution_ids":["https://openalex.org/I4210119991"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.23889859,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"169","last_page":"176"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9905999898910522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5720376968383789}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5720376968383789}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/slt61566.2024.10832323","is_oa":false,"landing_page_url":"https://doi.org/10.1109/slt61566.2024.10832323","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE Spoken Language Technology Workshop (SLT)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W1494198834","https://openalex.org/W1583837637","https://openalex.org/W2726515241","https://openalex.org/W2890964092","https://openalex.org/W3015468748","https://openalex.org/W3024869864","https://openalex.org/W3033529678","https://openalex.org/W3036601975","https://openalex.org/W3049256661","https://openalex.org/W3095173472","https://openalex.org/W3097777922","https://openalex.org/W3100460087","https://openalex.org/W3157506437","https://openalex.org/W3167533889","https://openalex.org/W3194203264","https://openalex.org/W3197580070","https://openalex.org/W3213029956","https://openalex.org/W4281492411","https://openalex.org/W4287704453","https://openalex.org/W4323066695","https://openalex.org/W4385245566","https://openalex.org/W4385570397","https://openalex.org/W4385807463","https://openalex.org/W4385822820","https://openalex.org/W4385822952","https://openalex.org/W4385823192","https://openalex.org/W4389326242","https://openalex.org/W4398230172","https://openalex.org/W4401609151","https://openalex.org/W4402112522","https://openalex.org/W6771467084","https://openalex.org/W6776048684","https://openalex.org/W6779163297","https://openalex.org/W6779709467","https://openalex.org/W6780218876","https://openalex.org/W6781533629","https://openalex.org/W6795140394","https://openalex.org/W6800388583","https://openalex.org/W6810673746","https://openalex.org/W6839026989","https://openalex.org/W6846807982","https://openalex.org/W6850218400","https://openalex.org/W6859298233","https://openalex.org/W6864424756","https://openalex.org/W6868207820"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Self-Supervised":[0],"Learning":[1],"(SSL)":[2],"has":[3],"proven":[4],"to":[5,48,112,123,130,138],"be":[6,49],"effective":[7],"in":[8,23,41,52],"various":[9],"domains,":[10],"including":[11],"speech":[12,43],"processing.":[13],"However,":[14],"SSL":[15,54,99],"is":[16,22],"computationally":[17],"and":[18,39,78,94,125],"memory":[19],"expensive.":[20],"This":[21],"part":[24],"due":[25],"the":[26,42,61,87,89,95,98],"quadratic":[27],"complexity":[28],"of":[29,63,91],"multi-head":[30],"self-attention":[31],"(MHSA).":[32],"Alternatives":[33],"for":[34,132],"MHSA":[35,65,113],"have":[36,46,71],"been":[37],"proposed":[38],"used":[40],"domain,":[44],"but":[45],"yet":[47],"investigated":[50],"properly":[51],"an":[53],"setting.":[55],"In":[56],"this":[57],"work,":[58],"we":[59],"study":[60],"effects":[62],"replacing":[64],"with":[66],"recent":[67],"state-of-the-art":[68],"alternatives":[69,107],"that":[70,104],"linear":[72,106],"complexity,":[73],"namely,":[74],"HyperMixing,":[75],"Fastformer,":[76],"SummaryMixing,":[77],"Mamba.":[79],"We":[80],"evaluate":[81],"these":[82,105],"methods":[83],"by":[84,120],"looking":[85],"at":[86],"speed,":[88],"amount":[90],"VRAM":[92,118],"consumed,":[93],"performance":[96,110],"on":[97,115],"MP3S":[100],"benchmark.":[101],"Results":[102],"show":[103],"maintain":[108],"competitive":[109],"compared":[111],"while,":[114],"average,":[116],"decreasing":[117],"consumption":[119],"around":[121],"20%":[122],"60%":[124],"increasing":[126],"speed":[127],"from":[128,136],"7%":[129],"65%":[131],"input":[133],"sequences":[134],"ranging":[135],"20":[137],"80":[139],"seconds.":[140]},"counts_by_year":[],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}