{"id":"https://openalex.org/W4407759759","doi":"https://doi.org/10.48550/arxiv.2502.12829","title":"KazMMLU: Evaluating Language Models on Kazakh, Russian, and Regional Knowledge of Kazakhstan","display_name":"KazMMLU: Evaluating Language Models on Kazakh, Russian, and Regional Knowledge of Kazakhstan","publication_year":2025,"publication_date":"2025-02-18","ids":{"openalex":"https://openalex.org/W4407759759","doi":"https://doi.org/10.48550/arxiv.2502.12829"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.12829","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.12829","pdf_url":"https://arxiv.org/pdf/2502.12829","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.12829","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5116335795","display_name":"Mukhammed Togmanov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Togmanov, Mukhammed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116335796","display_name":"Nurdaulet Mukhituly","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mukhituly, Nurdaulet","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075508196","display_name":"Diana Turmakhan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Turmakhan, Diana","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092029609","display_name":"Jonibek Mansurov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mansurov, Jonibek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115996950","display_name":"Maiya Goloburda","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Goloburda, Maiya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106361399","display_name":"Akhmed Sakip","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sakip, Akhmed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000261554","display_name":"Zhuohan Xie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xie, Zhuohan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100631221","display_name":"Yuxia Wang","orcid":"https://orcid.org/0000-0001-7676-999X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yuxia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039266918","display_name":"Bekassyl Syzdykov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Syzdykov, Bekassyl","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067626011","display_name":"Nurkhan Laiyk","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Laiyk, Nurkhan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112924039","display_name":"Alham Fikri Aji","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aji, Alham Fikri","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069779745","display_name":"Ekaterina Kochmar","orcid":"https://orcid.org/0000-0003-3328-1374"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kochmar, Ekaterina","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012055259","display_name":"Preslav Nakov","orcid":"https://orcid.org/0000-0002-3600-1510"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nakov, Preslav","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5065822589","display_name":"Fajri Koto","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Koto, Fajri","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9539999961853027,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9539999961853027,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/kazakh","display_name":"Kazakh","score":0.9913393259048462},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.34921038150787354},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.3262821435928345},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.2609061002731323},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.040035516023635864}],"concepts":[{"id":"https://openalex.org/C2781297163","wikidata":"https://www.wikidata.org/wiki/Q9252","display_name":"Kazakh","level":2,"score":0.9913393259048462},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.34921038150787354},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.3262821435928345},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2609061002731323},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.040035516023635864}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.12829","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.12829","pdf_url":"https://arxiv.org/pdf/2502.12829","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.12829","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.12829","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.12829","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.12829","pdf_url":"https://arxiv.org/pdf/2502.12829","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2363093443","https://openalex.org/W4206915444","https://openalex.org/W4388766676","https://openalex.org/W2353302181","https://openalex.org/W2362181825","https://openalex.org/W2320680668","https://openalex.org/W2901624366"],"abstract_inverted_index":{"Despite":[0],"having":[1],"a":[2],"population":[3],"of":[4,16,41,112,163],"twenty":[5],"million,":[6],"Kazakhstan's":[7,102],"culture":[8],"and":[9,44,75,83,89,97,106,120,140,161,167],"language":[10,18,22,32],"remain":[11],"underrepresented":[12],"in":[13,30,38,138],"the":[14,39,54,130],"field":[15],"natural":[17],"processing.":[19],"Although":[20],"large":[21],"models":[23,43,116,132],"(LLMs)":[24],"continue":[25],"to":[26,134,149],"advance":[27],"worldwide,":[28],"progress":[29],"Kazakh":[31,61,95,139],"has":[33],"been":[34],"limited,":[35],"as":[36,128],"seen":[37],"scarcity":[40],"dedicated":[42],"benchmark":[45],"evaluations.":[46],"To":[47],"address":[48],"this":[49],"gap,":[50],"we":[51],"introduce":[52],"KazMMLU,":[53],"first":[55],"MMLU-style":[56],"dataset":[57,92,156],"specifically":[58],"designed":[59],"for":[60,126],"language.":[62],"KazMMLU":[63],"comprises":[64],"23,000":[65],"questions":[66,96],"that":[67,154],"cover":[68],"various":[69],"educational":[70,81],"levels,":[71],"including":[72],"STEM,":[73],"humanities,":[74],"social":[76],"sciences,":[77],"sourced":[78],"from":[79],"authentic":[80],"materials":[82],"manually":[84],"validated":[85],"by":[86],"native":[87],"speakers":[88],"educators.":[90],"The":[91],"includes":[93],"10,969":[94],"12,031":[98],"Russian":[99],"questions,":[100],"reflecting":[101],"bilingual":[103],"education":[104],"system":[105],"rich":[107],"local":[108],"context.":[109],"Our":[110],"evaluation":[111],"several":[113],"state-of-the-art":[114],"multilingual":[115],"(Llama-3.1,":[117],"Qwen-2.5,":[118],"GPT-4,":[119],"DeepSeek":[121],"V3)":[122],"demonstrates":[123],"substantial":[124],"room":[125],"improvement,":[127],"even":[129],"best-performing":[131],"struggle":[133],"achieve":[135],"competitive":[136],"performance":[137,146],"Russian.":[141],"These":[142],"findings":[143],"underscore":[144],"significant":[145],"gaps":[147],"compared":[148],"high-resource":[150],"languages.":[151],"We":[152],"hope":[153],"our":[155],"will":[157,169],"enable":[158],"further":[159],"research":[160],"development":[162],"Kazakh-centric":[164],"LLMs.":[165],"Data":[166],"code":[168],"be":[170],"made":[171],"available":[172],"upon":[173],"acceptance.":[174]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}