{"id":"https://openalex.org/W7214639302","doi":"https://doi.org/10.48550/arxiv.2609.30484","title":"Do LLMs Understand Context? A Knowledge Graph-Based Evaluation Framework","display_name":"Do LLMs Understand Context? A Knowledge Graph-Based Evaluation Framework","publication_year":2026,"publication_date":"2026-09-24","ids":{"openalex":"https://openalex.org/W7214639302","doi":"https://doi.org/10.48550/arxiv.2609.30484"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.30484","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.30484","pdf_url":"https://arxiv.org/pdf/2609.30484","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.30484","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5151061859","display_name":"Subavarshana Arumugam","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I195740183","display_name":"University of Moratuwa","ror":"https://ror.org/0491f5305","country_code":"LK","type":"education","lineage":["https://openalex.org/I195740183"]}],"countries":["LK"],"is_corresponding":false,"raw_author_name":"Arumugam, Subavarshana","raw_affiliation_strings":["Department of Computer Science & Engineering , University of Moratuwa , Sri Lanka"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science & Engineering , University of Moratuwa , Sri Lanka","institution_ids":["https://openalex.org/I195740183"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5150964809","display_name":"Mamta Nallaretnam","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I195740183","display_name":"University of Moratuwa","ror":"https://ror.org/0491f5305","country_code":"LK","type":"education","lineage":["https://openalex.org/I195740183"]}],"countries":["LK"],"is_corresponding":false,"raw_author_name":"Nallaretnam, Mamta","raw_affiliation_strings":["Department of Computer Science & Engineering , University of Moratuwa , Sri Lanka","subavarshanaa. 21 , nallaretnam.21 ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science & Engineering , University of Moratuwa , Sri Lanka","institution_ids":["https://openalex.org/I195740183"]},{"raw_affiliation_string":"subavarshanaa. 21 , nallaretnam.21 ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5151084075","display_name":"Kithuni Wickramasinghe","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I195740183","display_name":"University of Moratuwa","ror":"https://ror.org/0491f5305","country_code":"LK","type":"education","lineage":["https://openalex.org/I195740183"]}],"countries":["LK"],"is_corresponding":false,"raw_author_name":"Wickramasinghe, Kithuni","raw_affiliation_strings":["Department of Computer Science & Engineering , University of Moratuwa , Sri Lanka"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science & Engineering , University of Moratuwa , Sri Lanka","institution_ids":["https://openalex.org/I195740183"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5150891986","display_name":"Chamath Gunapala","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I195740183","display_name":"University of Moratuwa","ror":"https://ror.org/0491f5305","country_code":"LK","type":"education","lineage":["https://openalex.org/I195740183"]}],"countries":["LK"],"is_corresponding":false,"raw_author_name":"Gunapala, Chamath","raw_affiliation_strings":["Department of Computer Science & Engineering , University of Moratuwa , Sri Lanka"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science & Engineering , University of Moratuwa , Sri Lanka","institution_ids":["https://openalex.org/I195740183"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115513445","display_name":"Pragatheeswaran Vipulanandan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I145608581","display_name":"University of Miami","ror":"https://ror.org/02dgjyy92","country_code":"US","type":"education","lineage":["https://openalex.org/I145608581"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vipulanandan, Pragatheeswaran","raw_affiliation_strings":["Department of Electrical and Computer Engineering , University of Miami , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering , University of Miami , USA","institution_ids":["https://openalex.org/I145608581"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018408973","display_name":"Kamal Premaratne","orcid":"https://orcid.org/0000-0001-5975-4403","observed_orcids":["https://orcid.org/0000-0001-5975-4403"]},"institutions":[{"id":"https://openalex.org/I145608581","display_name":"University of Miami","ror":"https://ror.org/02dgjyy92","country_code":"US","type":"education","lineage":["https://openalex.org/I145608581"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Premaratne, Kamal","raw_affiliation_strings":["Department of Electrical and Computer Engineering , University of Miami , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering , University of Miami , USA","institution_ids":["https://openalex.org/I145608581"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5049911731","display_name":"Uthayasanker Thayasivam","orcid":"https://orcid.org/0000-0002-3936-8174","observed_orcids":["https://orcid.org/0000-0002-3936-8174"]},"institutions":[{"id":"https://openalex.org/I145608581","display_name":"University of Miami","ror":"https://ror.org/02dgjyy92","country_code":"US","type":"education","lineage":["https://openalex.org/I145608581"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Thayasivam, Uthayasanker","raw_affiliation_strings":["Department of Electrical and Computer Engineering , University of Miami , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering , University of Miami , USA","institution_ids":["https://openalex.org/I145608581"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0044999998062849045,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0032999999821186066,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9670000076293945},{"id":"https://openalex.org/keywords/contextual-understanding","display_name":"contextual understanding","score":0.9700000286102295},{"id":"https://openalex.org/keywords/knowledge-graph-evaluation","display_name":"knowledge graph evaluation","score":0.722000002861023},{"id":"https://openalex.org/keywords/question-answering","display_name":"question answering","score":0.9340000152587891},{"id":"https://openalex.org/keywords/factual-consistency","display_name":"factual consistency","score":0.5329999923706055}],"concepts":[{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.7566999793052673},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.5834000110626221},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5679000020027161},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5389999747276306},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5227000117301941},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48829999566078186},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.4657999873161316},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.45249998569488525},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.42989999055862427},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.42750000953674316},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.37770000100135803},{"id":"https://openalex.org/C2777946921","wikidata":"https://www.wikidata.org/wiki/Q7449044","display_name":"Semantic analysis (machine learning)","level":2,"score":0.34850001335144043},{"id":"https://openalex.org/C52085439","wikidata":"https://www.wikidata.org/wiki/Q5165173","display_name":"Context analysis","level":3,"score":0.34380000829696655},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.326200008392334},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.31690001487731934},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.301800012588501},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.29159998893737793},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.287200003862381},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.2840000092983246},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.26829999685287476},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.267300009727478},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.2671999931335449},{"id":"https://openalex.org/C2776517306","wikidata":"https://www.wikidata.org/wiki/Q29017317","display_name":"Similarity measure","level":2,"score":0.25429999828338623},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.25049999356269836}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.30484","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.30484","pdf_url":"https://arxiv.org/pdf/2609.30484","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.30484","doi":"https://doi.org/10.48550/arxiv.2609.30484","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.30484","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.30484","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.30484","pdf_url":"https://arxiv.org/pdf/2609.30484","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G5277190103","display_name":"Tirer profit des donn\u00e9es externes pour une meilleure compr\u00e9hension et attribution causale des anomalies dans les syst\u00e8mes des r\u00e9seaux des eaux","funder_award_id":"ANR-25-CE23-2189","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214639302.pdf","grobid_xml":"https://content.openalex.org/works/W7214639302.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7212087654","https://openalex.org/W4412888099","https://openalex.org/W6979297322","https://openalex.org/W7134918936","https://openalex.org/W7162526791","https://openalex.org/W4391506308","https://openalex.org/W4416254814","https://openalex.org/W6910778003","https://openalex.org/W7130202338","https://openalex.org/W4404782876"],"abstract_inverted_index":{"While":[0],"large":[1],"language":[2],"models":[3,19],"(LLMs)":[4],"have":[5],"achieved":[6],"remarkable":[7],"linguistic":[8],"capabilities,":[9],"a":[10,47,53,86,111,135,145,150],"profound":[11],"question":[12,90],"lingers":[13],"at":[14,26,162],"their":[15],"core:":[16],"do":[17],"these":[18],"truly":[20],"comprehend":[21],"context":[22],"or":[23],"simply":[24,80,101],"excel":[25],"pattern":[27],"matching":[28],"on":[29],"an":[30],"unprecedented":[31],"scale?":[32],"Contextual":[33],"understanding":[34,122],"in":[35,89,123],"LLMs":[36],"refers":[37],"to":[38,41,61,126,156,182,192],"the":[39,163,186],"ability":[40],"correctly":[42],"extract":[43],"relevant":[44],"information":[45],"from":[46],"given":[48],"context,":[49],"integrate":[50],"it":[51,60],"into":[52,144],"coherent":[54],"internal":[55],"representation,":[56],"and":[57,65,78,141,158,189],"reason":[58],"over":[59,185],"produce":[62],"factually":[63],"consistent":[64],"contextually":[66,97],"grounded":[67,98],"responses.":[68],"However,":[69],"traditional":[70],"methods":[71],"such":[72],"as":[73],"BiLingual":[74],"Evaluation":[75],"Understudy":[76],"(BLEU)":[77],"perplexity":[79],"measure":[81,138],"surface-level":[82],"performance.":[83],"This":[84],"reveals":[85],"critical":[87],"gap":[88],"answering":[91],"(QA),":[92],"where":[93],"responses":[94],"must":[95],"be":[96],"rather":[99],"than":[100],"being":[102],"memorized":[103],"associations.":[104],"To":[105],"fill":[106],"this":[107,127],"void,":[108],"we":[109],"propose":[110],"novel":[112],"knowledge":[113],"graph":[114],"(KG)":[115],"based":[116],"evaluation":[117],"framework":[118,153],"for":[119,132],"LLM":[120],"contextual":[121],"QA.":[124],"Central":[125],"is":[128,154],"Semantic":[129],"Structural":[130],"Similarity":[131],"KGs":[133],"(S3KG),":[134],"hybrid":[136],"similarity":[137],"combining":[139],"structural":[140],"semantic":[142],"signals":[143],"single":[146],"score.":[147],"In":[148],"addition,":[149],"diagnostic":[151],"analysis":[152,168],"developed":[155],"identify":[157],"categorize":[159],"reasoning":[160],"errors":[161],"triplet":[164],"level,":[165],"enabling":[166],"fine-grained":[167],"of":[169,180],"model":[170],"failures.":[171],"Together,":[172],"across":[173],"nine":[174],"benchmarks,":[175],"S3KG":[176],"achieves":[177],"F1":[178],"gains":[179],"up":[181,191],"$+7.6$":[183],"points":[184],"strongest":[187],"baseline":[188],"AUROC":[190],"$0.973$.":[193]},"counts_by_year":[],"updated_date":"2026-11-06T12:05:46.655746","created_date":"2026-09-29T00:00:00"}