{"id":"https://openalex.org/W4412530513","doi":"https://doi.org/10.1007/s44443-025-00137-9","title":"SaudiCulture: A benchmark for evaluating large language models\u2019 cultural competence within Saudi Arabia","display_name":"SaudiCulture: A benchmark for evaluating large language models\u2019 cultural competence within Saudi Arabia","publication_year":2025,"publication_date":"2025-07-21","ids":{"openalex":"https://openalex.org/W4412530513","doi":"https://doi.org/10.1007/s44443-025-00137-9"},"language":"en","primary_location":{"id":"doi:10.1007/s44443-025-00137-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s44443-025-00137-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s44443-025-00137-9.pdf","source":{"id":"https://openalex.org/S2764955546","display_name":"Journal of King Saud University - Computer and Information Sciences","issn_l":"1319-1578","issn":["1319-1578","2213-1248"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of King Saud University Computer and Information Sciences","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://link.springer.com/content/pdf/10.1007/s44443-025-00137-9.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119027910","display_name":"Lama Ayash","orcid":"https://orcid.org/0000-0002-6268-9072"},"institutions":[{"id":"https://openalex.org/I82952536","display_name":"King Khalid University","ror":"https://ror.org/052kwzs30","country_code":"SA","type":"education","lineage":["https://openalex.org/I82952536"]}],"countries":["SA"],"is_corresponding":true,"raw_author_name":"Lama Ayash","raw_affiliation_strings":["Department of Computer Science, King Khalid University, Abha, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0002-6268-9072","affiliations":[{"raw_affiliation_string":"Department of Computer Science, King Khalid University, Abha, Saudi Arabia","institution_ids":["https://openalex.org/I82952536"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046502677","display_name":"Hassan Alhuzali","orcid":"https://orcid.org/0000-0002-0935-0774"},"institutions":[{"id":"https://openalex.org/I199693650","display_name":"Umm al-Qura University","ror":"https://ror.org/01xjqrm90","country_code":"SA","type":"education","lineage":["https://openalex.org/I199693650"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Hassan Alhuzali","raw_affiliation_strings":["Department of Computer Science & Artificial Intelligence, Umm Al-Qura University, Makkah, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0002-0935-0774","affiliations":[{"raw_affiliation_string":"Department of Computer Science & Artificial Intelligence, Umm Al-Qura University, Makkah, Saudi Arabia","institution_ids":["https://openalex.org/I199693650"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073657751","display_name":"Ashwag Alasmari","orcid":"https://orcid.org/0000-0002-2486-5979"},"institutions":[{"id":"https://openalex.org/I82952536","display_name":"King Khalid University","ror":"https://ror.org/052kwzs30","country_code":"SA","type":"education","lineage":["https://openalex.org/I82952536"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Ashwag Alasmari","raw_affiliation_strings":["Center for Artificial Intelligence (CAI), King Khalid University, Abha, Saudi Arabia","Department of Computer Science, King Khalid University, Abha, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0002-2486-5979","affiliations":[{"raw_affiliation_string":"Center for Artificial Intelligence (CAI), King Khalid University, Abha, Saudi Arabia","institution_ids":["https://openalex.org/I82952536"]},{"raw_affiliation_string":"Department of Computer Science, King Khalid University, Abha, Saudi Arabia","institution_ids":["https://openalex.org/I82952536"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5119027911","display_name":"Sultan Awwad Aloufi","orcid":null},"institutions":[{"id":"https://openalex.org/I23075662","display_name":"Taibah University","ror":"https://ror.org/01xv1nn60","country_code":"SA","type":"education","lineage":["https://openalex.org/I23075662"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Sultan Aloufi","raw_affiliation_strings":["Department of Arabic Language, Taibah University, Medina, Saudi Arabia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Arabic Language, Taibah University, Medina, Saudi Arabia","institution_ids":["https://openalex.org/I23075662"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5119027910"],"corresponding_institution_ids":["https://openalex.org/I82952536"],"apc_list":{"value":1350,"currency":"USD","value_usd":1350},"apc_paid":{"value":1350,"currency":"USD","value_usd":1350},"fwci":9.0927,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.97346452,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":"37","issue":"6","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13560","display_name":"Socioeconomic Development in MENA","score":0.9632999897003174,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13560","display_name":"Socioeconomic Development in MENA","score":0.9632999897003174,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13504","display_name":"African history and culture analysis","score":0.920799970626831,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13912","display_name":"Language, Linguistics, Cultural Analysis","score":0.9186000227928162,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5245164632797241},{"id":"https://openalex.org/keywords/competence","display_name":"Competence (human resources)","score":0.48780444264411926},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4222249686717987},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.3670170307159424},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3466063141822815},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.30916932225227356},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.1107955276966095},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.06918832659721375}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5245164632797241},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.48780444264411926},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4222249686717987},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3670170307159424},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3466063141822815},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.30916932225227356},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.1107955276966095},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.06918832659721375}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/s44443-025-00137-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s44443-025-00137-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s44443-025-00137-9.pdf","source":{"id":"https://openalex.org/S2764955546","display_name":"Journal of King Saud University - Computer and Information Sciences","issn_l":"1319-1578","issn":["1319-1578","2213-1248"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of King Saud University Computer and Information Sciences","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:2d3b2e705bd343568e747d86e1c9ee4f","is_oa":true,"landing_page_url":"https://doaj.org/article/2d3b2e705bd343568e747d86e1c9ee4f","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of King Saud University: Computer and Information Sciences, Vol 37, Iss 6, Pp 1-22 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1007/s44443-025-00137-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s44443-025-00137-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s44443-025-00137-9.pdf","source":{"id":"https://openalex.org/S2764955546","display_name":"Journal of King Saud University - Computer and Information Sciences","issn_l":"1319-1578","issn":["1319-1578","2213-1248"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of King Saud University Computer and Information Sciences","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10","score":0.4399999976158142}],"awards":[{"id":"https://openalex.org/G5466983459","display_name":null,"funder_award_id":"RGP1/337/45.","funder_id":"https://openalex.org/F33954902","funder_display_name":"Deanship of Scientific Research, King Khalid University"}],"funders":[{"id":"https://openalex.org/F33954902","display_name":"Deanship of Scientific Research, King Khalid University","ror":null},{"id":"https://openalex.org/F4320324047","display_name":"Umm Al-Qura University","ror":"https://ror.org/01xjqrm90"},{"id":"https://openalex.org/F4320324433","display_name":"King Khalid University","ror":"https://ror.org/052kwzs30"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412530513.pdf","grobid_xml":"https://content.openalex.org/works/W4412530513.grobid-xml"},"referenced_works_count":42,"referenced_works":["https://openalex.org/W1514905787","https://openalex.org/W2939803556","https://openalex.org/W4287889752","https://openalex.org/W4288058227","https://openalex.org/W4318919287","https://openalex.org/W4385571495","https://openalex.org/W4385572734","https://openalex.org/W4385812794","https://openalex.org/W4386200967","https://openalex.org/W4386384991","https://openalex.org/W4386566829","https://openalex.org/W4389519325","https://openalex.org/W4390723916","https://openalex.org/W4391296425","https://openalex.org/W4392039088","https://openalex.org/W4396608701","https://openalex.org/W4401042613","https://openalex.org/W4401042855","https://openalex.org/W4401043249","https://openalex.org/W4401043462","https://openalex.org/W4402521700","https://openalex.org/W4402670070","https://openalex.org/W4402671169","https://openalex.org/W4402671391","https://openalex.org/W4402671433","https://openalex.org/W4402671575","https://openalex.org/W4404782164","https://openalex.org/W4404782381","https://openalex.org/W4404782944","https://openalex.org/W4404783219","https://openalex.org/W4405738703","https://openalex.org/W4411630063","https://openalex.org/W4412073924","https://openalex.org/W6600009415","https://openalex.org/W6600103761","https://openalex.org/W6600820135","https://openalex.org/W6600850999","https://openalex.org/W6601289607","https://openalex.org/W6631274746","https://openalex.org/W6719981426","https://openalex.org/W6811838444","https://openalex.org/W6988892776"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W4246352526","https://openalex.org/W2121910908"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"remarkable":[6],"capabilities":[7],"in":[8,214,225,243],"natural":[9],"language":[10],"processing;":[11],"however,":[12],"they":[13],"often":[14],"struggle":[15],"to":[16,50,259,282],"accurately":[17,286],"capture":[18],"and":[19,39,61,82,105,122,139,155,164,285],"reflect":[20],"cultural":[21,41,53,62,98,137,165,232,245,262,289],"nuances.":[22],"This":[23],"research":[24],"addresses":[25],"this":[26],"challenge":[27],"by":[28,36,201],"focusing":[29],"on":[30,147],"Saudi":[31,65],"Arabia,":[32],"a":[33,46,69,94,109,270],"country":[34],"characterized":[35],"diverse":[37,288],"dialects":[38],"rich":[40],"traditions.":[42],"We":[43,143,264],"introduce":[44],"SaudiCulture,":[45],"novel":[47],"benchmark":[48],"designed":[49],"evaluate":[51],"the":[52,58,132,211,215,222,226,250,278],"competence":[54],"of":[55,64,72,97,115,252,280],"LLMs":[56,257,281],"within":[57],"distinct":[59],"geographical":[60,77],"contexts":[63],"Arabia.":[66],"SaudiCulture":[67,112,267],"is":[68],"comprehensive":[70],"dataset":[71,92,133],"questions":[73,86,114],"covering":[74],"five":[75],"major":[76],"regions\u2014West,":[78],"East,":[79],"South,":[80],"North,":[81],"Center\u2014along":[83],"with":[84,125,179,284],"general":[85],"applicable":[87],"across":[88,160],"all":[89,171],"regions.":[90],"The":[91],"encompasses":[93],"broad":[95],"spectrum":[96],"domains,":[99],"including":[100],"food,":[101],"clothing,":[102],"entertainment,":[103],"celebrations,":[104],"crafts.":[106],"To":[107],"ensure":[108],"rigorous":[110],"evaluation,":[111],"includes":[113],"varying":[116],"complexity,":[117],"such":[118],"as":[119,269],"open-ended,":[120],"single-choice,":[121],"multiple-choice":[123],"formats,":[124],"some":[126,196],"requiring":[127,187],"multiple":[128,188],"correct":[129,189],"answers.":[130],"Additionally,":[131,230],"distinguishes":[134],"between":[135],"common":[136],"knowledge":[138,255],"specialized":[140,181],"regional":[141],"aspects.":[142],"conduct":[144],"extensive":[145],"evaluations":[146],"six":[148],"LLMs\u2014GPT-4,":[149],"Llama":[150],"3.3,":[151],"FANAR,":[152],"Jais,":[153],"AceGPT":[154],"DeepSeek":[156],"\u2014analyzing":[157],"their":[158,261],"performance":[159,175],"different":[161],"question":[162],"types":[163],"contexts.":[166,290],"Our":[167],"findings":[168],"reveal":[169],"that":[170,194,266],"models":[172],"experience":[173],"significant":[174],"declines":[176],"when":[177],"faced":[178],"highly":[180],"or":[182],"region-specific":[183,254],"questions,":[184],"particularly":[185],"those":[186],"responses.":[190],"Furthermore,":[191],"we":[192],"observe":[193],"while":[195],"regions":[197],"are":[198,234],"better":[199],"understood":[200],"LLMs,":[202],"others":[203],"remain":[204],"largely":[205],"misrepresented.":[206],"For":[207],"instance,":[208],"GPT-4":[209],"achieves":[210],"highest":[212],"accuracy":[213,224],"western":[216],"region":[217,228],"(66%),":[218],"whereas":[219],"Jais":[220],"records":[221],"lowest":[223],"northern":[227],"(16%).":[229],"certain":[231],"categories":[233],"more":[235],"easily":[236],"identifiable":[237],"than":[238],"others,":[239],"further":[240],"highlighting":[241],"inconsistencies":[242],"LLMs\u2019":[244],"understanding.":[246],"These":[247],"results":[248],"emphasize":[249],"importance":[251],"incorporating":[253],"into":[256],"training":[258],"enhance":[260],"competence.":[263],"hope":[265],"serves":[268],"foundation":[271],"for":[272],"future":[273],"efforts":[274],"aimed":[275],"at":[276],"improving":[277],"ability":[279],"engage":[283],"represent":[287]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-13T06:13:01.061226","created_date":"2025-10-10T00:00:00"}