{"id":"https://openalex.org/W4412945357","doi":"https://doi.org/10.18653/v1/2025.acl-long.701","title":"KazMMLU: Evaluating Language Models on Kazakh, Russian, and Regional Knowledge of Kazakhstan","display_name":"KazMMLU: Evaluating Language Models on Kazakh, Russian, and Regional Knowledge of Kazakhstan","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412945357","doi":"https://doi.org/10.18653/v1/2025.acl-long.701"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.acl-long.701","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.701","pdf_url":"https://aclanthology.org/2025.acl-long.701.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.acl-long.701.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5116335795","display_name":"Mukhammed Togmanov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mukhammed Togmanov","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116335796","display_name":"Nurdaulet Mukhituly","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nurdaulet Mukhituly","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075508196","display_name":"Diana Turmakhan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Diana Turmakhan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092029609","display_name":"Jonibek Mansurov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jonibek Mansurov","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115996950","display_name":"Maiya Goloburda","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maiya Goloburda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106361399","display_name":"Akhmed Sakip","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Akhmed Sakip","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000261554","display_name":"Zhuohan Xie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhuohan Xie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100631230","display_name":"Yuxia Wang","orcid":"https://orcid.org/0009-0003-1804-9021"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuxia Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039266918","display_name":"Bekassyl Syzdykov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bekassyl Syzdykov","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067626011","display_name":"Nurkhan Laiyk","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nurkhan Laiyk","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112924039","display_name":"Alham Fikri Aji","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alham Fikri Aji","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069779745","display_name":"Ekaterina Kochmar","orcid":"https://orcid.org/0000-0003-3328-1374"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ekaterina Kochmar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012055259","display_name":"Preslav Nakov","orcid":"https://orcid.org/0000-0002-3600-1510"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Preslav Nakov","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5065822589","display_name":"Fajri Koto","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fajri Koto","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":11.7111,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.98720881,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"14403","last_page":"14416"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9491000175476074,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9491000175476074,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/kazakh","display_name":"Kazakh","score":0.9886281490325928},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.595483124256134},{"id":"https://openalex.org/keywords/russian-language","display_name":"Russian language","score":0.5254026055335999},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.38936150074005127},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.3045995235443115}],"concepts":[{"id":"https://openalex.org/C2781297163","wikidata":"https://www.wikidata.org/wiki/Q9252","display_name":"Kazakh","level":2,"score":0.9886281490325928},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.595483124256134},{"id":"https://openalex.org/C2993748607","wikidata":"https://www.wikidata.org/wiki/Q7737","display_name":"Russian language","level":2,"score":0.5254026055335999},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38936150074005127},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3045995235443115},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.acl-long.701","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.701","pdf_url":"https://aclanthology.org/2025.acl-long.701.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.acl-long.701","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.701","pdf_url":"https://aclanthology.org/2025.acl-long.701.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7400000095367432}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412945357.pdf","grobid_xml":"https://content.openalex.org/works/W4412945357.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2363093443","https://openalex.org/W4206915444","https://openalex.org/W4388766676","https://openalex.org/W2353302181","https://openalex.org/W2362181825","https://openalex.org/W2320680668","https://openalex.org/W2901624366"],"abstract_inverted_index":{"Mukhammed":[0],"Togmanov,":[1],"Nurdaulet":[2],"Mukhituly,":[3],"Diana":[4],"Turmakhan,":[5],"Jonibek":[6],"Mansurov,":[7],"Maiya":[8],"Goloburda,":[9],"Akhmed":[10],"Sakip,":[11],"Zhuohan":[12],"Xie,":[13],"Yuxia":[14],"Wang,":[15],"Bekassyl":[16],"Syzdykov,":[17],"Nurkhan":[18],"Laiyk,":[19],"Alham":[20],"Fikri":[21],"Aji,":[22],"Ekaterina":[23],"Kochmar,":[24],"Preslav":[25],"Nakov,":[26],"Fajri":[27],"Koto.":[28],"Proceedings":[29],"of":[30,35],"the":[31,36],"63rd":[32],"Annual":[33],"Meeting":[34],"Association":[37],"for":[38],"Computational":[39],"Linguistics":[40],"(Volume":[41],"1:":[42],"Long":[43],"Papers).":[44],"2025.":[45]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":3}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}