{"id":"https://openalex.org/W7140088165","doi":"https://doi.org/10.18653/v1/2026.eacl-long.46","title":"Cetvel: A Unified Benchmark for Evaluating Language Understanding, Generation and Cultural Capacity of LLMs for Turkish","display_name":"Cetvel: A Unified Benchmark for Evaluating Language Understanding, Generation and Cultural Capacity of LLMs for Turkish","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7140088165","doi":"https://doi.org/10.18653/v1/2026.eacl-long.46"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2026.eacl-long.46","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.eacl-long.46","pdf_url":"https://aclanthology.org/2026.eacl-long.46.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 19th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.eacl-long.46.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5130400592","display_name":"Yakup Abrek Er","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yakup Abrek Er","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123913538","display_name":"Ilker Kesen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ilker Kesen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124672675","display_name":"G\u00f6zde G\u00fcl Sahin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"G\u00f6zde G\u00fcl \u015eahin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5004736226","display_name":"Aykut Erdem","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aykut Erdem","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.38415982,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1052","last_page":"1085"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.45500001311302185,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.45500001311302185,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.15469999611377716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12881","display_name":"linguistics and terminology studies","score":0.056299999356269836,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/turkish","display_name":"Turkish","score":0.6970999836921692},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4871000051498413},{"id":"https://openalex.org/keywords/cultural-diversity","display_name":"Cultural diversity","score":0.25929999351501465}],"concepts":[{"id":"https://openalex.org/C2781121862","wikidata":"https://www.wikidata.org/wiki/Q256","display_name":"Turkish","level":2,"score":0.6970999836921692},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4871000051498413},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4794999957084656},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33180001378059387},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.30469998717308044},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.27730000019073486},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2623000144958496},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.26170000433921814},{"id":"https://openalex.org/C125209646","wikidata":"https://www.wikidata.org/wiki/Q1338878","display_name":"Cultural diversity","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.25529998540878296},{"id":"https://openalex.org/C107826830","wikidata":"https://www.wikidata.org/wiki/Q929380","display_name":"Environmental resource management","level":1,"score":0.25360000133514404},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.2517000138759613},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.25099998712539673}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.18653/v1/2026.eacl-long.46","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.eacl-long.46","pdf_url":"https://aclanthology.org/2026.eacl-long.46.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 19th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/fb6c6ea9-8dda-4336-9fa3-9056f1ee9a28","is_oa":true,"landing_page_url":"https://researchprofiles.ku.dk/da/publications/fb6c6ea9-8dda-4336-9fa3-9056f1ee9a28","pdf_url":null,"source":{"id":"https://openalex.org/S4306401983","display_name":"Research at the University of Copenhagen (University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I124055696","host_organization_name":"University of Copenhagen","host_organization_lineage":["https://openalex.org/I124055696"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Er , A , Kesen , I , \u015eahin , G G & Erdem , A 2026 , CETVEL : A Unified Benchmark for Evaluating Language Understanding, Generation and Cultural Capacity of LLMs for Turkish . in V Demberg , K Inui & L Marquez Villodre (eds) , Proceedings of the 19th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers) . Association for Computational Linguistics (ACL) , pp. 1052-1085 , 19th Conference of the European Chapter of the Association for Computational Linguistics, EACL 2026 , Rabat , Morocco , 24/03/2026 . https://doi.org/10.18653/v1/2026.eacl-long.46","raw_type":"contributionToPeriodical"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.eacl-long.46","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.eacl-long.46","pdf_url":"https://aclanthology.org/2026.eacl-long.46.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 19th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.723331868648529,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G3800857974","display_name":"Democratize Trustworthy and Efficient Large Language Model Technology for Europe","funder_award_id":"101135671","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7140088165.pdf","grobid_xml":"https://content.openalex.org/works/W7140088165.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0],"introduce":[1],"CETVEL,":[2],"a":[3,33,140],"comprehensive":[4,141],"benchmark":[5],"designed":[6],"to":[7,85,104],"evaluate":[8,80],"large":[9],"language":[10],"models":[11,100,108],"(LLMs)":[12],"in":[13,74,135,155],"Turkish.Existing":[14],"Turkish":[15,52,75],"benchmarks":[16],"often":[17],"lack":[18],"either":[19],"task":[20],"diversity":[21],"or":[22,26,106],"culturally":[23,143],"relevant":[24],"content,":[25],"both.CETVEL":[27],"addresses":[28],"these":[29],"gaps":[30],"by":[31],"combining":[32],"broad":[34],"range":[35],"of":[36,51,153],"both":[37],"discriminative":[38,134],"and":[39,48,70,77,92,111,128,142,151],"generative":[40],"tasks":[41,56,62,122],"ensuring":[42],"content":[43],"that":[44,97,121],"reflects":[45],"the":[46,117,149],"linguistic":[47],"cultural":[49],"richness":[50],"language.CETVEL":[53],"covers":[54],"23":[55],"grouped":[57],"into":[58],"seven":[59],"categories,":[60],"including":[61],"such":[63,123],"as":[64,124],"grammatical":[65,125],"error":[66,126],"correction,":[67],"machine":[68],"translation,":[69],"question":[71,130],"answering":[72,131],"rooted":[73],"history":[76],"idiomatic":[78],"language.We":[79],"33":[81],"open-weight":[82],"LLMs":[83,154],"(up":[84],"70B":[86],"parameters)":[87],"covering":[88],"different":[89],"model":[90,137],"families":[91],"instruction":[93],"paradigms.Our":[94],"experiments":[95],"reveal":[96],"Turkish-centric":[98],"instruction-tuned":[99],"generally":[101],"underperform":[102],"relative":[103],"multilingual":[105],"general-purpose":[107],"(e.g.Llama":[109],"3":[110],"Mistral),":[112],"despite":[113],"being":[114],"tailored":[115],"for":[116,147],"language.Moreover,":[118],"we":[119],"show":[120],"correction":[127],"extractive":[129],"are":[132],"particularly":[133],"differentiating":[136],"capabilities.CETVEL":[138],"offers":[139],"grounded":[144],"evaluation":[145],"suite":[146],"advancing":[148],"development":[150],"assessment":[152],"Turkish.":[156]},"counts_by_year":[],"updated_date":"2026-06-14T07:44:22.658603","created_date":"2026-03-24T00:00:00"}