{"id":"https://openalex.org/W4416037765","doi":"https://doi.org/10.18653/v1/2025.arabicnlp-sharedtasks.107","title":"PalmX 2025: The First Shared Task on Benchmarking LLMs on Arabic and Islamic Culture","display_name":"PalmX 2025: The First Shared Task on Benchmarking LLMs on Arabic and Islamic Culture","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416037765","doi":"https://doi.org/10.18653/v1/2025.arabicnlp-sharedtasks.107"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2025.arabicnlp-sharedtasks.107","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.arabicnlp-sharedtasks.107","pdf_url":"https://aclanthology.org/2025.arabicnlp-sharedtasks.107.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Third Arabic Natural Language Processing Conference: Shared Tasks","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.arabicnlp-sharedtasks.107.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054154345","display_name":"Fakhraddin Alwajih","orcid":"https://orcid.org/0000-0002-8530-4243"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fakhraddin Alwajih","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024996988","display_name":"Abdellah El Mekki","orcid":"https://orcid.org/0000-0002-7394-3611"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abdellah El Mekki","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102924560","display_name":"Hamdy Mubarak","orcid":"https://orcid.org/0000-0002-9051-6240"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hamdy Mubarak","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021151663","display_name":"Majd Hawasly","orcid":"https://orcid.org/0000-0003-1823-5580"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Majd Hawasly","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035596051","display_name":"Asmaa Mohamed","orcid":"https://orcid.org/0000-0002-9531-1834"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abubakr Mohamed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5004629670","display_name":"Muhammad Abdul-Mageed","orcid":"https://orcid.org/0000-0002-8590-2040"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Muhammad Abdul-Mageed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"774","last_page":"789"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13523","display_name":"Mathematics, Computing, and Information Processing","score":0.27070000767707825,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13523","display_name":"Mathematics, Computing, and Information Processing","score":0.27070000767707825,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0746999979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14330","display_name":"Library Science and Information Systems","score":0.06849999725818634,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6499999761581421},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6438999772071838},{"id":"https://openalex.org/keywords/islam","display_name":"Islam","score":0.6225000023841858},{"id":"https://openalex.org/keywords/arabic","display_name":"Arabic","score":0.525600016117096},{"id":"https://openalex.org/keywords/islamic-culture","display_name":"Islamic culture","score":0.5109000205993652}],"concepts":[{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6499999761581421},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6438999772071838},{"id":"https://openalex.org/C4445939","wikidata":"https://www.wikidata.org/wiki/Q432","display_name":"Islam","level":2,"score":0.6225000023841858},{"id":"https://openalex.org/C96455323","wikidata":"https://www.wikidata.org/wiki/Q13955","display_name":"Arabic","level":2,"score":0.525600016117096},{"id":"https://openalex.org/C2776132995","wikidata":"https://www.wikidata.org/wiki/Q1340372","display_name":"Islamic culture","level":3,"score":0.5109000205993652},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.42170000076293945},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.3644999861717224},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.3506999909877777},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.272599995136261},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2703999876976013},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2565999925136566},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.2549000084400177}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.arabicnlp-sharedtasks.107","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.arabicnlp-sharedtasks.107","pdf_url":"https://aclanthology.org/2025.arabicnlp-sharedtasks.107.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Third Arabic Natural Language Processing Conference: Shared Tasks","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.arabicnlp-sharedtasks.107","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.arabicnlp-sharedtasks.107","pdf_url":"https://aclanthology.org/2025.arabicnlp-sharedtasks.107.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Third Arabic Natural Language Processing Conference: Shared Tasks","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2980479518","display_name":null,"funder_award_id":"RGPIN-2018","funder_id":"https://openalex.org/F4320334593","funder_display_name":"Natural Sciences and Engineering Research Council of Canada"},{"id":"https://openalex.org/G4097713262","display_name":"Conflict in Nova Scotia:  women organizations and institutionalized restorative justice","funder_award_id":"37771","funder_id":"https://openalex.org/F4320334617","funder_display_name":"Social Sciences and Humanities Research Council of Canada"},{"id":"https://openalex.org/G8828985329","display_name":null,"funder_award_id":"895-2020-1004","funder_id":"https://openalex.org/F4320334617","funder_display_name":"Social Sciences and Humanities Research Council of Canada"}],"funders":[{"id":"https://openalex.org/F4320320994","display_name":"Canada Research Chairs","ror":"https://ror.org/0517h6h17"},{"id":"https://openalex.org/F4320331257","display_name":"Alliance de recherche num\u00e9rique du Canada","ror":"https://ror.org/010r6td27"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"},{"id":"https://openalex.org/F4320334617","display_name":"Social Sciences and Humanities Research Council of Canada","ror":"https://ror.org/04j5jqy92"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416037765.pdf","grobid_xml":"https://content.openalex.org/works/W4416037765.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"inherently":[4],"reflect":[5],"the":[6,22,42,84,91,196,205,224,235,256,260],"vast":[7],"data":[8,17,208],"distributions":[9],"they":[10],"encounter":[11],"during":[12],"their":[13,60],"pre-training":[14],"phase.As":[15],"this":[16,77,215],"is":[18,25,56,101],"predominantly":[19],"sourced":[20],"from":[21,138],"web,":[23],"there":[24],"a":[26,47,53,124,218],"high":[27],"chance":[28],"it":[29],"will":[30],"be":[31,213],"skewed":[32],"towards":[33],"high-resourced":[34],"languages":[35],"and":[36,64,118,135,154,162,188,198,230,242],"cultures,":[37],"such":[38],"as":[39,195],"those":[40],"of":[41,50,62,94,103,127,183,207,226,234],"West.Consequently,":[43],"LLMs":[44,95],"often":[45],"exhibit":[46],"diminished":[48],"understanding":[49],"certain":[51],"communities,":[52],"gap":[54],"that":[55,169,239],"particularly":[57],"evident":[58],"in":[59,96,110,160,259],"knowledge":[61,245],"Arabic":[63,113,116,232],"Islamic":[65,120,191],"cultures.This":[66],"issue":[67],"becomes":[68],"even":[69],"more":[70,227],"pronounced":[71],"with":[72,147],"increasingly":[73],"under-represented":[74],"topics.To":[75],"address":[76],"critical":[78],"challenge,":[79],"we":[80],"introduce":[81],"PalmX":[82],"2025,":[83],"first":[85],"shared":[86,236,257],"task":[87,100,237,258],"designed":[88],"to":[89,212,222,248,252,254],"benchmark":[90,216],"cultural":[92,186,241],"competence":[93],"these":[97],"specific":[98],"domains.The":[99],"composed":[102],"two":[104],"subtasks":[105,122],"featuring":[106],"multiplechoice":[107],"questions":[108,187],"(MCQs)":[109],"Modern":[111],"Standard":[112],"(MSA):":[114],"General":[115,119],"Culture":[117],"Culture.These":[121],"cover":[123],"wide":[125],"range":[126],"topics,":[128],"including":[129],"traditions,":[130],"food,":[131],"history,":[132],"religious":[133,244],"practices,":[134],"language":[136],"expressions":[137],"across":[139],"22":[140],"Arab":[141],"countries.The":[142],"initiative":[143],"drew":[144],"considerable":[145],"interest,":[146],"26":[148],"teams":[149],"registering":[150],"for":[151,156],"Subtask":[152,157],"1":[153],"19":[155],"2,":[158],"culminating":[159],"nine":[161],"six":[163],"valid":[164],"submissions,":[165],"respectively.Our":[166],"findings":[167],"re-veal":[168],"task-specific":[170],"fine-tuning":[171],"substantially":[172],"boosts":[173],"performance":[174],"over":[175],"baseline":[176],"models.The":[177],"top-performing":[178],"systems":[179],"achieved":[180],"an":[181],"accuracy":[182],"72.15%":[184],"on":[185,190],"84.22%":[189],"knowledge.Parameter-efficient":[192],"finetuning":[193],"emerged":[194],"predominant":[197],"most":[199],"effective":[200],"approach":[201],"among":[202],"participants,":[203],"while":[204],"utility":[206],"augmentation":[209],"was":[210],"found":[211],"domain-dependent.Ultimately,":[214],"provides":[217],"crucial,":[219],"standardized":[220],"framework":[221],"guide":[223],"development":[225],"culturally":[228],"grounded":[229],"competent":[231],"LLMs.Results":[233],"demonstrate":[238],"general":[240,243],"remain":[246],"challenging":[247],"LLMs,":[249],"motivating":[250],"us":[251],"continue":[253],"offer":[255],"future.":[261]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-11-08T00:00:00"}