{"id":"https://openalex.org/W7131832579","doi":"https://doi.org/10.48550/arxiv.2602.22827","title":"TARAZ: Persian Short-Answer Question Benchmark for Cultural Evaluation of Language Models","display_name":"TARAZ: Persian Short-Answer Question Benchmark for Cultural Evaluation of Language Models","publication_year":2026,"publication_date":"2026-02-26","ids":{"openalex":"https://openalex.org/W7131832579","doi":"https://doi.org/10.48550/arxiv.2602.22827"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2602.22827","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.22827","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2602.22827","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5116335556","display_name":"Reihaneh Iranmanesh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Iranmanesh, Reihaneh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127067908","display_name":"Saeedeh Davoudi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Davoudi, Saeedeh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127076575","display_name":"Pasha Abrishamchian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abrishamchian, Pasha","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127292478","display_name":"Ophir Frieder","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Frieder, Ophir","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5036610566","display_name":"Nazli Goharian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Goharian, Nazli","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.38370001316070557,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.38370001316070557,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.325300008058548,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.04230000078678131,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/persian","display_name":"Persian","score":0.566100001335144},{"id":"https://openalex.org/keywords/normalization","display_name":"Normalization (sociology)","score":0.5594000220298767},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5120999813079834},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.44920000433921814},{"id":"https://openalex.org/keywords/evaluation-methods","display_name":"Evaluation methods","score":0.36419999599456787},{"id":"https://openalex.org/keywords/semantic-similarity","display_name":"Semantic similarity","score":0.35659998655319214},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.3179999887943268}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6991999745368958},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6108999848365784},{"id":"https://openalex.org/C2776527531","wikidata":"https://www.wikidata.org/wiki/Q9168","display_name":"Persian","level":2,"score":0.566100001335144},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.5594000220298767},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.554099977016449},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5120999813079834},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.44920000433921814},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.36419999599456787},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.35659998655319214},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.3179999887943268},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.28679999709129333},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.28380000591278076},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2827000021934509},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.2754000127315521},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.27300000190734863},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.2646999955177307},{"id":"https://openalex.org/C2777146004","wikidata":"https://www.wikidata.org/wiki/Q14949826","display_name":"CLARITY","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.25760000944137573},{"id":"https://openalex.org/C157486923","wikidata":"https://www.wikidata.org/wiki/Q1376436","display_name":"String (physics)","level":2,"score":0.2567000091075897},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.25589999556541443}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2602.22827","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.22827","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2602.22827","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.22827","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7666863203048706}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"This":[0],"paper":[1],"presents":[2],"a":[3,44,54,147],"comprehensive":[4],"evaluation":[5,47,71,90,110,132,153],"framework":[6,42],"for":[7,139,150],"assessing":[8],"the":[9,114,135],"cultural":[10,21,141],"competence":[11],"of":[12,72],"large":[13],"language":[14],"models":[15,78],"(LLMs)":[16],"in":[17,143],"Persian.":[18],"Existing":[19],"Persian":[20,83,144],"benchmarks":[22],"rely":[23],"predominantly":[24],"on":[25],"multiple-choice":[26],"formats":[27],"and":[28,38,57,76,145],"English-centric":[29],"metrics":[30],"that":[31,48,87,103,113],"fail":[32],"to":[33,97],"capture":[34],"Persian's":[35],"morphological":[36,51],"complexity":[37],"semantic":[39,58,116],"nuance.":[40],"Our":[41,108],"introduces":[43],"Persian-specific":[45],"short-answer":[46],"combines":[49],"rule-based":[50],"normalization":[52],"with":[53,122],"hybrid":[55,89],"syntactic":[56],"similarity":[59,117],"module,":[60],"enabling":[61],"robust":[62],"soft-match":[63],"scoring":[64,92],"beyond":[65],"exact":[66],"string":[67],"overlap.":[68],"Through":[69],"systematic":[70],"15":[73],"state-of-the-art":[74],"open-":[75],"closed-source":[77],"across":[79],"three":[80],"culturally":[81],"grounded":[82],"datasets,":[84],"we":[85],"demonstrate":[86],"our":[88,131],"improves":[91],"consistency":[93],"by":[94,100],"+10":[95],"compared":[96],"exact-match":[98],"baselines":[99],"capturing":[101],"meaning":[102],"surface-level":[104],"methods":[105],"cannot":[106],"detect.":[107],"human":[109,123],"further":[111],"confirms":[112],"proposed":[115],"metric":[118],"achieves":[119],"higher":[120],"agreement":[121],"judgments":[124],"than":[125],"LLM-based":[126],"judges.":[127],"We":[128],"publicly":[129],"release":[130],"framework,":[133],"providing":[134],"first":[136],"standardized":[137],"benchmark":[138],"measuring":[140],"understanding":[142],"establishing":[146],"reproducible":[148],"foundation":[149],"cross-cultural":[151],"LLM":[152],"research.":[154]},"counts_by_year":[],"updated_date":"2026-08-16T07:02:28.622633","created_date":"2026-02-28T00:00:00"}