{"id":"https://openalex.org/W4402684121","doi":"https://doi.org/10.18653/v1/2024.findings-acl.671","title":"CMMLU: Measuring massive multitask language understanding in Chinese","display_name":"CMMLU: Measuring massive multitask language understanding in Chinese","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4402684121","doi":"https://doi.org/10.18653/v1/2024.findings-acl.671"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2024.findings-acl.671","doi":"https://doi.org/10.18653/v1/2024.findings-acl.671","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-acl.671","pdf_url":"https://aclanthology.org/2024.findings-acl.671.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics ACL 2024","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2024.findings-acl.671.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100742674","display_name":"Haonan Li","orcid":"https://orcid.org/0000-0002-9258-4466","observed_orcids":["https://orcid.org/0000-0002-9258-4466"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haonan Li","raw_affiliation_strings":["College Engineering Hydrology College Actuarial Science Astronomy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College Engineering Hydrology College Actuarial Science Astronomy","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037162594","display_name":"Yixuan Zhang","orcid":"https://orcid.org/0000-0002-7412-4669","observed_orcids":["https://orcid.org/0000-0002-7412-4669"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yixuan Zhang","raw_affiliation_strings":["College Engineering Hydrology College Actuarial Science Astronomy","Virology Machine Learning High School Physics High School Mathematics High School Chemistry High School Biology Genetics Elementary Mathematics Electrical Engineering Conceptual Physics Computer Science"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College Engineering Hydrology College Actuarial Science Astronomy","institution_ids":[]},{"raw_affiliation_string":"Virology Machine Learning High School Physics High School Mathematics High School Chemistry High School Biology Genetics Elementary Mathematics Electrical Engineering Conceptual Physics Computer Science","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065822589","display_name":"Fajri Koto","orcid":"https://orcid.org/0000-0002-3659-6761","observed_orcids":["https://orcid.org/0000-0002-3659-6761"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fajri Koto","raw_affiliation_strings":["College Engineering Hydrology College Actuarial Science Astronomy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College Engineering Hydrology College Actuarial Science Astronomy","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100300571","display_name":"Yifei Yang","orcid":"https://orcid.org/0000-0001-5666-2407","observed_orcids":["https://orcid.org/0000-0001-5666-2407"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yifei Yang","raw_affiliation_strings":["Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036050911","display_name":"Hai Zhao","orcid":"https://orcid.org/0000-0001-7290-0487","observed_orcids":["https://orcid.org/0000-0001-7290-0487"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hai Zhao","raw_affiliation_strings":["Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041448669","display_name":"Yeyun Gong","orcid":"https://orcid.org/0000-0001-9954-9674","observed_orcids":["https://orcid.org/0000-0001-9954-9674"]},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yeyun Gong","raw_affiliation_strings":["Microsoft Research Asia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Asia","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Nan Duan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Nan Duan","raw_affiliation_strings":["Microsoft Research Asia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Asia","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103085805","display_name":"Timothy Baldwin","orcid":"https://orcid.org/0000-0002-4445-1386","observed_orcids":["https://orcid.org/0000-0002-4445-1386","https://orcid.org/0000-0003-4525-6950"]},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"The University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Timothy Baldwin","raw_affiliation_strings":["The University of Melbourne"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Melbourne","institution_ids":["https://openalex.org/I165779595"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":70.722,"has_fulltext":true,"cited_by_count":480,"citation_normalized_percentile":{"value":0.99930924,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"11260","last_page":"11285"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9663000106811523,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9663000106811523,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.01600000075995922,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.013299999758601189,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.8420000076293945},{"id":"https://openalex.org/keywords/chinese-language-models","display_name":"Chinese language models","score":0.4740000069141388},{"id":"https://openalex.org/keywords/multilingual-llms","display_name":"multilingual LLMs","score":0.6510000228881836}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7833197116851807},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4988369941711426},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36382633447647095}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2024.findings-acl.671","doi":"https://doi.org/10.18653/v1/2024.findings-acl.671","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-acl.671","pdf_url":"https://aclanthology.org/2024.findings-acl.671.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics ACL 2024","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2306.09212","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.09212","pdf_url":"https://arxiv.org/pdf/2306.09212","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2306.09212","doi":"https://doi.org/10.48550/arxiv.2306.09212","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2306.09212","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2024.findings-acl.671","doi":"https://doi.org/10.18653/v1/2024.findings-acl.671","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-acl.671","pdf_url":"https://aclanthology.org/2024.findings-acl.671.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics ACL 2024","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4402684121.pdf","grobid_xml":"https://content.openalex.org/works/W4402684121.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2794325560","https://openalex.org/W2946609015","https://openalex.org/W2963323070","https://openalex.org/W2963995027","https://openalex.org/W3025064182","https://openalex.org/W3114651185","https://openalex.org/W3161374759","https://openalex.org/W3201174429","https://openalex.org/W3210904070","https://openalex.org/W4229005866","https://openalex.org/W4308760226","https://openalex.org/W4320003957","https://openalex.org/W4322718191","https://openalex.org/W4327810158","https://openalex.org/W4366327277","https://openalex.org/W4367061104","https://openalex.org/W4376653782","https://openalex.org/W4377161536","https://openalex.org/W4378465252","https://openalex.org/W4379468930","https://openalex.org/W4383173625","https://openalex.org/W4385373704","https://openalex.org/W4385571124","https://openalex.org/W4389157038","https://openalex.org/W4411630326","https://openalex.org/W6893809404"],"related_works":["https://openalex.org/W4393399727","https://openalex.org/W4396820591","https://openalex.org/W4391158820","https://openalex.org/W7131812994","https://openalex.org/W4377161536","https://openalex.org/W7017787997","https://openalex.org/W4367061104","https://openalex.org/W4399151467","https://openalex.org/W4392488491","https://openalex.org/W4401024141"],"abstract_inverted_index":{"As":[0],"the":[1,31,50,89,104,125,129],"capabilities":[2,105,133],"of":[3,33,56,84,106,134],"large":[4,135],"language":[5,136],"models":[6,137],"(LLMs)":[7],"continue":[8],"to":[9,23,80,112],"advance,":[10],"evaluating":[11,128],"their":[12,66],"performance":[13,67,117],"is":[14,88,98],"becoming":[15],"more":[16,19,57],"important":[17],"and":[18,49,62,71,118,131],"challenging.This":[20],"paper":[21],"aims":[22],"address":[24],"this":[25],"issue":[26],"for":[27,92,101,121,138],"Mandarin":[28],"Chinese":[29,37,63,93],"in":[30,103,127],"form":[32],"CMMLU,":[34],"a":[35,53],"comprehensive":[36],"benchmark":[38],"that":[39,75,96],"covers":[40],"various":[41],"subjects,":[42],"including":[43],"natural":[44],"sciences,":[45,47],"social":[46],"engineering,":[48],"humanities.We":[51],"conduct":[52,109],"thorough":[54],"evaluation":[55],"than":[58],"20":[59],"contemporary":[60],"multilingual":[61],"LLMs,":[64],"assessing":[65],"across":[68],"different":[69],"subjects":[70],"settings.The":[72],"results":[73],"reveal":[74],"most":[76],"existing":[77],"LLMs":[78],"struggle":[79],"achieve":[81],"an":[82],"accuracy":[83],"even":[85],"60%,":[86],"which":[87],"pass":[90],"mark":[91],"exams.This":[94],"highlights":[95],"there":[97],"substantial":[99],"room":[100],"improvement":[102],"LLMs.Additionally,":[107],"we":[108],"extensive":[110],"experiments":[111],"identify":[113],"factors":[114],"impacting":[115],"model":[116],"propose":[119],"directions":[120],"enhancing":[122],"LLMs.CMMLU":[123],"fills":[124],"gap":[126],"knowledge":[130],"reasoning":[132],"Chinese.":[139]},"counts_by_year":[{"year":2026,"cited_by_count":119},{"year":2025,"cited_by_count":214},{"year":2024,"cited_by_count":130},{"year":2023,"cited_by_count":17}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2024-09-21T00:00:00"}