{"id":"https://openalex.org/W7135037855","doi":"https://doi.org/10.1016/j.jjimei.2026.100404","title":"Benchmarking Large Language Models on Arabic parsing","display_name":"Benchmarking Large Language Models on Arabic parsing","publication_year":2026,"publication_date":"2026-03-11","ids":{"openalex":"https://openalex.org/W7135037855","doi":"https://doi.org/10.1016/j.jjimei.2026.100404"},"language":"en","primary_location":{"id":"doi:10.1016/j.jjimei.2026.100404","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jjimei.2026.100404","pdf_url":null,"source":{"id":"https://openalex.org/S4210226089","display_name":"International Journal of Information Management Data Insights","issn_l":"2667-0968","issn":["2667-0968"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Information Management Data Insights","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1016/j.jjimei.2026.100404","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128890312","display_name":"Kamal Al-Sabahi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kamal Al-Sabahi","raw_affiliation_strings":["College of Banking and Financial Studies, Sultanate of Oman"],"raw_orcid":"https://orcid.org/0000-0001-5554-9533","affiliations":[{"raw_affiliation_string":"College of Banking and Financial Studies, Sultanate of Oman","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128875845","display_name":"Amer Fael Mohammed Balhaf","orcid":null},"institutions":[{"id":"https://openalex.org/I116005404","display_name":"Ashiya University","ror":"https://ror.org/056fhh809","country_code":"JP","type":"education","lineage":["https://openalex.org/I116005404"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Amer Fael Mohammed Balhaf","raw_affiliation_strings":["ASharqiyah University, Sultanate of Oman"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ASharqiyah University, Sultanate of Oman","institution_ids":["https://openalex.org/I116005404"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128903749","display_name":"Mohamed Moheb Zagloul Al Shamy","orcid":null},"institutions":[{"id":"https://openalex.org/I4403386702","display_name":"University of Technology and Applied Sciences-Suhar","ror":"https://ror.org/018g8cj68","country_code":"OM","type":"education","lineage":["https://openalex.org/I4403386702","https://openalex.org/I4405258755"]}],"countries":["OM"],"is_corresponding":false,"raw_author_name":"Mohamed Moheb Zagloul Al Shamy","raw_affiliation_strings":["University of Technology and Applied Sciences, Sultanate of Oman"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Technology and Applied Sciences, Sultanate of Oman","institution_ids":["https://openalex.org/I4403386702"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067972997","display_name":"Majjed Al-Qatf","orcid":null},"institutions":[{"id":"https://openalex.org/I188760350","display_name":"Ollscoil na Gaillimhe \u2013 University of Galway","ror":"https://ror.org/03bea9k73","country_code":"IE","type":"education","lineage":["https://openalex.org/I188760350"]}],"countries":["IE"],"is_corresponding":true,"raw_author_name":"Majjed Al-Qatf","raw_affiliation_strings":["Insight Centre for Data Analytics, University of Galway, Ireland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Insight Centre for Data Analytics, University of Galway, Ireland","institution_ids":["https://openalex.org/I188760350"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072066119","display_name":"Kang Yang","orcid":"https://orcid.org/0000-0002-1944-4328"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kang Yang","raw_affiliation_strings":["National University of Defense Technology, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Defense Technology, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004581951","display_name":"Abdulrahman Al-badwi","orcid":"https://orcid.org/0000-0002-6203-8747"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Abdulrahman Al-Badwi","raw_affiliation_strings":["Central South University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University, China","institution_ids":["https://openalex.org/I139660479"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5067972997"],"corresponding_institution_ids":["https://openalex.org/I188760350"],"apc_list":{"value":1500,"currency":"USD","value_usd":1500},"apc_paid":{"value":1500,"currency":"USD","value_usd":1500},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.32112025,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"6","issue":"1","first_page":"100404","last_page":"100404"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.7727000117301941,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.7727000117301941,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.05000000074505806,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.0272000003606081,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/parsing","display_name":"Parsing","score":0.8048999905586243},{"id":"https://openalex.org/keywords/arabic","display_name":"Arabic","score":0.5307999849319458},{"id":"https://openalex.org/keywords/bottom-up-parsing","display_name":"Bottom-up parsing","score":0.5084999799728394},{"id":"https://openalex.org/keywords/ambiguity","display_name":"Ambiguity","score":0.4587000012397766},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.45100000500679016},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.3828999996185303},{"id":"https://openalex.org/keywords/computational-linguistics","display_name":"Computational linguistics","score":0.34380000829696655}],"concepts":[{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.8048999905586243},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7806000113487244},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.7649999856948853},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6464999914169312},{"id":"https://openalex.org/C96455323","wikidata":"https://www.wikidata.org/wiki/Q13955","display_name":"Arabic","level":2,"score":0.5307999849319458},{"id":"https://openalex.org/C60690694","wikidata":"https://www.wikidata.org/wiki/Q894902","display_name":"Bottom-up parsing","level":4,"score":0.5084999799728394},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.4587000012397766},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.45100000500679016},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.3828999996185303},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.34380000829696655},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.3384999930858612},{"id":"https://openalex.org/C147547768","wikidata":"https://www.wikidata.org/wiki/Q3113342","display_name":"S-attributed grammar","level":3,"score":0.32280001044273376},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.30480000376701355},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.30399999022483826},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.2870999872684479},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C2778243841","wikidata":"https://www.wikidata.org/wiki/Q56467","display_name":"Modern Standard Arabic","level":3,"score":0.2754000127315521},{"id":"https://openalex.org/C148524875","wikidata":"https://www.wikidata.org/wiki/Q6975395","display_name":"F1 score","level":2,"score":0.2741999924182892},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.2653000056743622}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1016/j.jjimei.2026.100404","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jjimei.2026.100404","pdf_url":null,"source":{"id":"https://openalex.org/S4210226089","display_name":"International Journal of Information Management Data Insights","issn_l":"2667-0968","issn":["2667-0968"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Information Management Data Insights","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:80725f0752b84714a718cc85290f13ad","is_oa":true,"landing_page_url":"https://doaj.org/article/80725f0752b84714a718cc85290f13ad","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"International Journal of Information Management Data Insights, Vol 6, Iss 1, Pp 100404- (2026)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1016/j.jjimei.2026.100404","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jjimei.2026.100404","pdf_url":null,"source":{"id":"https://openalex.org/S4210226089","display_name":"International Journal of Information Management Data Insights","issn_l":"2667-0968","issn":["2667-0968"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Information Management Data Insights","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.771158754825592,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":6,"referenced_works":["https://openalex.org/W2904213695","https://openalex.org/W2996728628","https://openalex.org/W3185341429","https://openalex.org/W4292622500","https://openalex.org/W4361006021","https://openalex.org/W4367626326"],"related_works":[],"abstract_inverted_index":{"Parsing":[0],"Arabic":[1,37,164,187,200,216],"sentences,":[2],"specifically":[3],"i":[4,38],"c":[5,39],"r\u0101b":[6,40],",":[7],"poses":[8],"unique":[9],"challenges":[10,151],"due":[11],"to":[12,116,144],"the":[13,27,74,123],"language\u2019s":[14],"intricate":[15],"morphology,":[16],"diverse":[17],"syntactic":[18,60,201],"structures,":[19],"and":[20,61,84,94,121,134,155,174,192,202,213],"rich":[21],"contextual":[22],"nuances.":[23],"This":[24],"study":[25],"evaluates":[26],"performance":[28,57,138],"of":[29,114,125,130,185,208],"leading":[30],"general-purpose":[31,168],"Large":[32],"Language":[33],"Models":[34],"(LLMs)":[35],"in":[36,118,167,224],"parsing":[41,166,204,221],"using":[42],"a":[43,160],"novel":[44],"human-annotated":[45],"dataset,":[46],"systematically":[47],"covering":[48],"various":[49],"grammatical":[50,165],"phenomena.":[51],"A":[52],"tailored":[53],"evaluation":[54,175,197],"framework":[55,198],"assesses":[56],"across":[58,109,189],"detailed":[59],"morphological":[62,203],"features.":[63],"Under":[64],"matched":[65],"multi-shot":[66],"prompting":[67,105,219],"(basis":[68],"for":[69,163],"cross-model":[70],"comparisons),":[71],"Claude-3.5-Sonnet":[72],"achieved":[73],"highest":[75],"overall":[76],"F1":[77],"score":[78],"(0.84),":[79],"followed":[80],"by":[81],"GPT-4o":[82],"(0.83)":[83],"Gemini-1.5-Pro":[85],"(0.77).":[86],"Conversely,":[87],"less":[88],"advanced":[89],"models":[90,132],"such":[91],"as":[92],"Claude2.1":[93],"GPT-3.5-turbo":[95],"struggled":[96],"with":[97],"complex":[98,119,225],"constructions,":[99],"highlighting":[100],"persistent":[101],"linguistic":[102],"limitations.":[103],"Multi-shot":[104,218],"substantially":[106],"improved":[107],"accuracy":[108,222],"proprietary":[110,145],"models,":[111],"yielding":[112],"improvements":[113],"up":[115],"18%":[117],"categories":[120,191,226],"underscoring":[122],"value":[124],"in-context":[126],"learning.":[127],"Additionally,":[128],"evaluations":[129],"open-source":[131],"(DeepSeek-chat-v3-0324":[133],"LLaMA-4-scout)":[135],"established":[136],"baseline":[137],"levels":[139],"confirming":[140],"substantial":[141],"gaps":[142],"compared":[143],"models.":[146],"The":[147],"findings":[148],"reveal":[149],"ongoing":[150],"like":[152],"diacritic":[153],"sensitivity":[154],"semantic":[156],"ambiguity":[157],"while":[158],"establishing":[159],"robust":[161],"benchmark":[162],"LLMs.":[169],"All":[170],"resources":[171],"(dataset,":[172],"codebase,":[173],"outputs)":[176],"are":[177],"available":[178],"at":[179],"https://github.com/alsabahi2030/Arabic-LLM-Parsing":[180],".":[181],"\u2022":[182,195,205,217],"Human-annotated":[183],"dataset":[184],"1100":[186],"sentences":[188],"11":[190],"34":[193],"subtypes":[194],"Tailored":[196],"assessing":[199],"Comparative":[206],"analysis":[207],"Claude,":[209],"GPT,":[210],"Gemini,":[211],"DeepSeek,":[212],"LLaMA-4":[214],"on":[215],"boosts":[220],"considerably":[223]},"counts_by_year":[],"updated_date":"2026-06-14T07:44:22.658603","created_date":"2026-03-13T00:00:00"}