{"id":"https://openalex.org/W7148294870","doi":"https://doi.org/10.1109/asru65441.2025.11434707","title":"mSTEB: Massively Multilingual Evaluation of LLMs on Speech and Text Tasks","display_name":"mSTEB: Massively Multilingual Evaluation of LLMs on Speech and Text Tasks","publication_year":2025,"publication_date":"2025-12-06","ids":{"openalex":"https://openalex.org/W7148294870","doi":"https://doi.org/10.1109/asru65441.2025.11434707"},"language":null,"primary_location":{"id":"doi:10.1109/asru65441.2025.11434707","is_oa":false,"landing_page_url":"https://doi.org/10.1109/asru65441.2025.11434707","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120777433","display_name":"Luel Hagos Beyene","orcid":null},"institutions":[{"id":"https://openalex.org/I4210138097","display_name":"Riverbank Local Redevelopment Authority","ror":"https://ror.org/04qz58m35","country_code":"US","type":"government","lineage":["https://openalex.org/I4210138097"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Luel Hagos Beyene","raw_affiliation_strings":["AIMS RIC"],"affiliations":[{"raw_affiliation_string":"AIMS RIC","institution_ids":["https://openalex.org/I4210138097"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056119295","display_name":"Vivek Verma","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Vivek Verma","raw_affiliation_strings":["Mila - Quebec AI Institute"],"affiliations":[{"raw_affiliation_string":"Mila - Quebec AI Institute","institution_ids":["https://openalex.org/I4210164802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132812474","display_name":"Min Ma","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Min Ma","raw_affiliation_strings":["Google DeepMind"],"affiliations":[{"raw_affiliation_string":"Google DeepMind","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055769964","display_name":"Jesujoba O. Alabi","orcid":null},"institutions":[{"id":"https://openalex.org/I91712215","display_name":"Saarland University","ror":"https://ror.org/01jdpyv68","country_code":"DE","type":"education","lineage":["https://openalex.org/I91712215"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Jesujoba O. Alabi","raw_affiliation_strings":["Saarland University"],"affiliations":[{"raw_affiliation_string":"Saarland University","institution_ids":["https://openalex.org/I91712215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011479082","display_name":"Fabian David Schmidt","orcid":"https://orcid.org/0009-0007-9841-2941"},"institutions":[{"id":"https://openalex.org/I25974101","display_name":"University of W\u00fcrzburg","ror":"https://ror.org/00fbnyb24","country_code":"DE","type":"education","lineage":["https://openalex.org/I25974101"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Fabian David Schmidt","raw_affiliation_strings":["University of Würzburg"],"affiliations":[{"raw_affiliation_string":"University of Würzburg","institution_ids":["https://openalex.org/I25974101"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130262546","display_name":"Joyce Nakatumba-Nabende","orcid":null},"institutions":[{"id":"https://openalex.org/I72227227","display_name":"Makerere University","ror":"https://ror.org/03dmz0111","country_code":"UG","type":"education","lineage":["https://openalex.org/I72227227"]}],"countries":["UG"],"is_corresponding":false,"raw_author_name":"Joyce Nakatumba-Nabende","raw_affiliation_strings":["Makerere University"],"affiliations":[{"raw_affiliation_string":"Makerere University","institution_ids":["https://openalex.org/I72227227"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5132797311","display_name":"David Ifeoluwa Adelani","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"David Ifeoluwa Adelani","raw_affiliation_strings":["Mila - Quebec AI Institute"],"affiliations":[{"raw_affiliation_string":"Mila - Quebec AI Institute","institution_ids":["https://openalex.org/I4210164802"]}]}],"institutions":[],"countries_distinct_count":5,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5120777433"],"corresponding_institution_ids":["https://openalex.org/I4210138097"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.87338771,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.18459999561309814,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.18459999561309814,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1145000010728836,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.08100000023841858,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5011000037193298},{"id":"https://openalex.org/keywords/spoken-language","display_name":"Spoken language","score":0.4934000074863434},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.31940001249313354},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.2671000063419342},{"id":"https://openalex.org/keywords/government","display_name":"Government (linguistics)","score":0.24320000410079956}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6601999998092651},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5011000037193298},{"id":"https://openalex.org/C2776230583","wikidata":"https://www.wikidata.org/wiki/Q1322198","display_name":"Spoken language","level":2,"score":0.4934000074863434},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.39660000801086426},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.36230000853538513},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3587000072002411},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.31940001249313354},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.2671000063419342},{"id":"https://openalex.org/C2778137410","wikidata":"https://www.wikidata.org/wiki/Q2732820","display_name":"Government (linguistics)","level":2,"score":0.24320000410079956},{"id":"https://openalex.org/C27548731","wikidata":"https://www.wikidata.org/wiki/Q88272","display_name":"Investment (military)","level":3,"score":0.2386000007390976}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/asru65441.2025.11434707","is_oa":false,"landing_page_url":"https://doi.org/10.1109/asru65441.2025.11434707","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.7027207612991333,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320309327","display_name":"Google","ror":"https://ror.org/00njsd438"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W2963846996","https://openalex.org/W3169369929","https://openalex.org/W3213029956","https://openalex.org/W4387321091","https://openalex.org/W4389524378","https://openalex.org/W4401043498","https://openalex.org/W4402111264","https://openalex.org/W4402667012","https://openalex.org/W4402684123","https://openalex.org/W4404781972","https://openalex.org/W4404782746","https://openalex.org/W4406385636","https://openalex.org/W4408345687","https://openalex.org/W4411119839","https://openalex.org/W4412887872","https://openalex.org/W7133200369"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"models":[2,101],"(LLMs)":[3],"have":[4],"demonstrated":[5],"impressive":[6],"performance":[7,59,87,118],"on":[8,62,78],"a":[9,30,53,63,114],"wide":[10,64,115],"range":[11,65],"of":[12,60,66,88],"tasks,":[13],"including":[14],"in":[15,117,128,144],"multimodal":[16],"settings":[17],"such":[18,91,102],"as":[19,92,103],"speech.":[20],"However,":[21],"their":[22,142],"evaluation":[23,41,112],"is":[24,38,138],"often":[25],"limited":[26],"to":[27,56,140],"English":[28],"and":[29,75,81,96,98,107,121,130],"few":[31],"high-resource":[32,120],"languages.":[33],"For":[34],"low-resource":[35,122],"languages,":[36,123],"there":[37],"no":[39],"standardized":[40],"benchmark.":[42],"In":[43],"this":[44,48],"paper,":[45],"we":[46],"address":[47,141],"gap":[49,116],"by":[50],"introducing":[51],"mSTEB,":[52],"new":[54],"benchmark":[55],"evaluate":[57],"the":[58,86],"LLMs":[61,90,145],"tasks":[67,77],"covering":[68],"language":[69],"identification,":[70],"text":[71,82],"classification,":[72],"question":[73],"answering,":[74],"translation":[76],"both":[79],"speech":[80],"modalities.":[83],"We":[84],"evaluated":[85],"leading":[89],"Gemini":[93],"2.0":[94],"Flash":[95],"GPT-4o":[97],"state-of-the-art":[99],"open":[100],"Qwen":[104],"2":[105],"Audio":[106],"Gemma":[108],"327":[109],"B.":[110],"Our":[111,132],"shows":[113],"between":[119],"especially":[124],"for":[125],"languages":[126],"spoken":[127],"Africa":[129],"Americas/Oceania.":[131],"findings":[133],"show":[134],"that":[135],"more":[136],"investment":[137],"needed":[139],"under-representation":[143],"coverage.":[146]},"counts_by_year":[],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2026-04-03T00:00:00"}