{"id":"https://openalex.org/W7164801418","doi":"https://doi.org/10.1145/3816713.3816760","title":"Lilit to Ramakien: A Five-Work Benchmark for LLM Understanding of Classical Thai Literature","display_name":"Lilit to Ramakien: A Five-Work Benchmark for LLM Understanding of Classical Thai Literature","publication_year":2026,"publication_date":"2026-06-15","ids":{"openalex":"https://openalex.org/W7164801418","doi":"https://doi.org/10.1145/3816713.3816760"},"language":null,"primary_location":{"id":"doi:10.1145/3816713.3816760","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3816713.3816760","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 14th International Conference on Advances in Information Technology","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3816713.3816760","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036316550","display_name":"Tuul Triyason","orcid":"https://orcid.org/0000-0002-5970-7751"},"institutions":[{"id":"https://openalex.org/I79246082","display_name":"Thaksin University","ror":"https://ror.org/00t2prd39","country_code":"TH","type":"education","lineage":["https://openalex.org/I79246082"]}],"countries":["TH"],"is_corresponding":true,"raw_author_name":"Tuul Triyason","raw_affiliation_strings":["School of Information Technology, Thungkru, Bangkok, Thailand"],"raw_orcid":"https://orcid.org/0000-0002-5970-7751","affiliations":[{"raw_affiliation_string":"School of Information Technology, Thungkru, Bangkok, Thailand","institution_ids":["https://openalex.org/I79246082"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5036316550"],"corresponding_institution_ids":["https://openalex.org/I79246082"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.94555695,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.11630000174045563,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.11630000174045563,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.10239999741315842,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13002","display_name":"Southeast Asian Sociopolitical Studies","score":0.09220000356435776,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/frontier","display_name":"Frontier","score":0.6685000061988831},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5658000111579895},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.4433000087738037},{"id":"https://openalex.org/keywords/poetry","display_name":"Poetry","score":0.4253999888896942},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.3702000081539154},{"id":"https://openalex.org/keywords/divergence","display_name":"Divergence (linguistics)","score":0.36169999837875366},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.36059999465942383},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.3603000044822693}],"concepts":[{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.6685000061988831},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5658000111579895},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.4433000087738037},{"id":"https://openalex.org/C164913051","wikidata":"https://www.wikidata.org/wiki/Q482","display_name":"Poetry","level":2,"score":0.4253999888896942},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.40790000557899475},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4034999907016754},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3702000081539154},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.36959999799728394},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.3693000078201294},{"id":"https://openalex.org/C207390915","wikidata":"https://www.wikidata.org/wiki/Q1230525","display_name":"Divergence (linguistics)","level":2,"score":0.36169999837875366},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.36059999465942383},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.3603000044822693},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.33320000767707825},{"id":"https://openalex.org/C2778304055","wikidata":"https://www.wikidata.org/wiki/Q657474","display_name":"Beijing","level":3,"score":0.3246999979019165},{"id":"https://openalex.org/C2778753569","wikidata":"https://www.wikidata.org/wiki/Q1960395","display_name":"Span (engineering)","level":2,"score":0.31619998812675476},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.2994999885559082},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.29589998722076416},{"id":"https://openalex.org/C2779525943","wikidata":"https://www.wikidata.org/wiki/Q1187300","display_name":"Grammaticality","level":3,"score":0.2946000099182129},{"id":"https://openalex.org/C2780428219","wikidata":"https://www.wikidata.org/wiki/Q16952335","display_name":"Cover (algebra)","level":2,"score":0.28519999980926514},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2800000011920929},{"id":"https://openalex.org/C93692415","wikidata":"https://www.wikidata.org/wiki/Q1502030","display_name":"Thematic map","level":2,"score":0.2745000123977661},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.2694999873638153},{"id":"https://openalex.org/C2780861071","wikidata":"https://www.wikidata.org/wiki/Q1062934","display_name":"Character (mathematics)","level":2,"score":0.2648000121116638},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.26010000705718994},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.2590000033378601},{"id":"https://openalex.org/C125209646","wikidata":"https://www.wikidata.org/wiki/Q1338878","display_name":"Cultural diversity","level":2,"score":0.25589999556541443},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.2533999979496002}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3816713.3816760","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3816713.3816760","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 14th International Conference on Advances in Information Technology","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3816713.3816760","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3816713.3816760","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 14th International Conference on Advances in Information Technology","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.8215574622154236}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":3,"referenced_works":["https://openalex.org/W2963748441","https://openalex.org/W2993640618","https://openalex.org/W3173544627"],"related_works":[],"abstract_inverted_index":{"Classical":[0],"Thai":[1,47,67],"literature":[2],"spans":[3],"over":[4],"five":[5,56,113,257],"centuries":[6],"of":[7,65,164],"poetic":[8],"tradition,":[9],"yet":[10],"the":[11,62,66,94,120,132,173,195,199,236,251],"extent":[12],"to":[13],"which":[14],"modern":[15],"large":[16],"language":[17],"models":[18],"(LLMs)":[19],"can":[20],"engage":[21],"with":[22,147,259],"its":[23],"archaic":[24],"vocabulary,":[25],"complex":[26],"verse":[27,103,158],"forms,":[28],"and":[29,88,96,99,108,127,131,137,144,247,282],"layered":[30],"cultural":[31],"philosophy":[32],"remains":[33],"largely":[34,218],"unknown.":[35],"We":[36,111],"introduce":[37],"a":[38,148,161,185],"pilot":[39],"five-work":[40],"benchmark":[41,284],"for":[42,286],"evaluating":[43],"LLMs":[44,115],"on":[45,202,266,277],"classical":[46,68,278],"literary":[48,289],"comprehension,":[49],"comprising":[50],"57":[51],"question-answering":[52],"pairs":[53],"drawn":[54],"from":[55],"canonical":[57],"works":[58,92,171],"that":[59,187],"collectively":[60],"represent":[61],"full":[63],"breadth":[64],"tradition:":[69],"Lilit":[70,74],"Phra":[71],"Lo":[72],"(\u0e25\u0e34\u0e25\u0e34\u0e15\u0e1e\u0e23\u0e30\u0e25\u0e2d),":[73],"Taleng":[75],"Phai":[76],"(\u0e25\u0e34\u0e25\u0e34\u0e15\u0e15\u0e30\u0e40\u0e25\u0e07\u0e1e\u0e48\u0e32\u0e22),":[77],"Khun":[78,80],"Chang":[79],"Phaen":[81],"(\u0e02\u0e38\u0e19\u0e0a\u0e49\u0e32\u0e07\u0e02\u0e38\u0e19\u0e41\u0e1c\u0e19),":[82],"Nirat":[83],"Phu":[84],"Khao":[85],"Thong":[86],"(\u0e19\u0e34\u0e23\u0e32\u0e28\u0e20\u0e39\u0e40\u0e02\u0e32\u0e17\u0e2d\u0e07),":[87],"Ramakien":[89],"(\u0e23\u0e32\u0e21\u0e40\u0e01\u0e35\u0e22\u0e23\u0e15\u0e34\u0e4c).":[90],"These":[91,272],"span":[93],"Ayutthaya":[95],"Rattanakosin":[97],"periods":[98],"cover":[100],"four":[101],"distinct":[102],"forms:":[104],"lilit,":[105],"sepha,":[106],"klon,":[107],"bot":[109],"lakhon.":[110],"evaluated":[112],"leading":[114],"across":[116,255],"two":[117],"operational":[118],"tiers,":[119],"Frontier":[121,200],"tier":[122,134,197,201],"(Claude":[123],"Opus":[124],"4.7,":[125],"GPT-5.5,":[126],"Gemini":[128,138,220],"3":[129,139,221],"Pro)":[130],"Efficiency":[133,196],"(GPT-4o":[135],"mini":[136],"Flash),":[140],"using":[141],"keyword-overlap":[142],"accuracy":[143,176,265],"BERTScore":[145],"F1":[146,211,232],"multilingual":[149],"backbone.":[150],"Our":[151],"analysis":[152],"reveals":[153],"three":[154],"principal":[155],"findings.":[156],"First,":[157],"form":[159],"is":[160],"stronger":[162],"predictor":[163],"model":[165,261,280],"performance":[166],"than":[167],"historical":[168],"era:":[169],"lilit-form":[170],"achieve":[172],"highest":[174],"keyword":[175,224],"(0.512),":[177],"while":[178],"klon":[179],"travel":[180],"poetry":[181],"scores":[182],"lowest":[183],"(0.344),":[184],"pattern":[186],"cannot":[188],"be":[189],"explained":[190],"by":[191,219],"chronology":[192],"alone.":[193],"Second,":[194],"outperforms":[198],"both":[203],"aggregate":[204],"metrics":[205],"(keyword":[206],"gap":[207,212],"+":[208,213],"0.148,":[209],"BERT":[210],"0.123),":[214],"an":[215],"effect":[216],"driven":[217],"Pro\u2019s":[222],"near-zero":[223],"recall":[225],"(0.049)":[226],"despite":[227],"moderate":[228],"semantic":[229],"similarity":[230],"(BERT":[231],"=":[233,243],"0.643),":[234],"yielding":[235],"largest":[237],"lexical-semantic":[238],"divergence":[239],"score":[240],"observed":[241],"(\u0394":[242],"+0.594).":[244],"Third,":[245],"thematic":[246],"philosophical":[248],"questions":[249],"remain":[250],"hardest":[252],"task":[253],"type":[254],"all":[256],"works,":[258],"no":[260],"consistently":[262],"achieving":[263],"high":[264],"Buddhist":[267],"or":[268],"monarchical":[269],"ethics":[270],"questions.":[271],"findings":[273],"motivate":[274],"future":[275],"work":[276],"language-specific":[279],"adaptation":[281],"multi-annotator":[283],"construction":[285],"Southeast":[287],"Asian":[288],"heritage.":[290]},"counts_by_year":[],"updated_date":"2026-06-16T07:37:23.134862","created_date":"2026-06-16T00:00:00"}