{"id":"https://openalex.org/W4415257529","doi":"https://doi.org/10.1145/3770855.3817527","title":"CTIConnect: A Benchmark for Retrieval-Augmented LLMs over Heterogeneous Cyber Threat Intelligence","display_name":"CTIConnect: A Benchmark for Retrieval-Augmented LLMs over Heterogeneous Cyber Threat Intelligence","publication_year":2026,"publication_date":"2026-08-06","ids":{"openalex":"https://openalex.org/W4415257529","doi":"https://doi.org/10.1145/3770855.3817527"},"language":"en","primary_location":{"id":"doi:10.1145/3770855.3817527","doi":"https://doi.org/10.1145/3770855.3817527","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3770855.3817527","pdf_url":null,"source":{"id":"https://openalex.org/S4363608767","display_name":"ACM SIGKDD Conference on Knowledge Discovery and Data Mining (KDD)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3770855.3817527","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102664784","display_name":"Yutong Cheng","orcid":"https://orcid.org/0009-0003-8641-8079","observed_orcids":["https://orcid.org/0009-0003-8641-8079"]},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yutong Cheng","raw_affiliation_strings":["Virginia Tech, Blacksburg, VA, USA"],"raw_orcid":"https://orcid.org/0009-0003-8641-8079","affiliations":[{"raw_affiliation_string":"Virginia Tech, Blacksburg, VA, USA","institution_ids":["https://openalex.org/I859038795"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100355692","display_name":"Yang Liu","orcid":"https://orcid.org/0000-0001-7300-9215","observed_orcids":["https://orcid.org/0000-0001-7300-9215","https://orcid.org/0000-0002-6276-1468","https://orcid.org/0000-0001-9704-8120","https://orcid.org/0009-0003-6607-861X"]},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang Liu","raw_affiliation_strings":["Virginia Tech, Blacksburg, VA, USA"],"raw_orcid":"https://orcid.org/0009-0003-6607-861X","affiliations":[{"raw_affiliation_string":"Virginia Tech, Blacksburg, VA, USA","institution_ids":["https://openalex.org/I859038795"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119247038","display_name":"Changze Li","orcid":"https://orcid.org/0009-0009-5708-0287","observed_orcids":["https://orcid.org/0009-0009-5708-0287","https://orcid.org/0009-0000-5024-3009"]},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Changze Li","raw_affiliation_strings":["Virginia Tech, Blacksburg, VA, USA"],"raw_orcid":"https://orcid.org/0009-0000-5024-3009","affiliations":[{"raw_affiliation_string":"Virginia Tech, Blacksburg, VA, USA","institution_ids":["https://openalex.org/I859038795"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019426968","display_name":"Dawn Xiaodong Song","orcid":"https://orcid.org/0000-0001-9745-6802","observed_orcids":["https://orcid.org/0000-0001-9745-6802"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dawn Song","raw_affiliation_strings":["University of California, Berkeley, Berkeley, CA, USA"],"raw_orcid":"https://orcid.org/0000-0001-9745-6802","affiliations":[{"raw_affiliation_string":"University of California, Berkeley, Berkeley, CA, USA","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102796137","display_name":"Peng Gao","orcid":"https://orcid.org/0000-0002-9342-9863","observed_orcids":["https://orcid.org/0000-0002-9342-9863"]},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peng Gao","raw_affiliation_strings":["Virginia Tech, Blacksburg, VA, USA"],"raw_orcid":"https://orcid.org/0000-0002-9342-9863","affiliations":[{"raw_affiliation_string":"Virginia Tech, Blacksburg, VA, USA","institution_ids":["https://openalex.org/I859038795"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.5469,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.92758705,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"8741","last_page":"8752"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.47920000553131104,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.47920000553131104,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.41019999980926514,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.026399999856948853,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cyber-threat-intelligence","display_name":"cyber threat intelligence","score":0.9900000095367432},{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.5899999737739563},{"id":"https://openalex.org/keywords/heterogeneous-knowledge-sources","display_name":"heterogeneous knowledge sources","score":0.7450000047683716},{"id":"https://openalex.org/keywords/entity-linking","display_name":"entity linking","score":0.902999997138977},{"id":"https://openalex.org/keywords/use-of-evidence","display_name":"evidence use","score":0.47099998593330383},{"id":"https://openalex.org/keywords/cve","display_name":"CVE","score":0.32600000500679016},{"id":"https://openalex.org/keywords/cwe","display_name":"CWE","score":0.7519999742507935},{"id":"https://openalex.org/keywords/capec","display_name":"CAPEC","score":0.9390000104904175},{"id":"https://openalex.org/keywords/mitre-att-ck","display_name":"MITRE ATT&CK","score":0.9980000257492065}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7802000045776367},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5866000056266785},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5663999915122986},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.46149998903274536},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.415800005197525},{"id":"https://openalex.org/C2780428219","wikidata":"https://www.wikidata.org/wiki/Q16952335","display_name":"Cover (algebra)","level":2,"score":0.38940000534057617},{"id":"https://openalex.org/C517642484","wikidata":"https://www.wikidata.org/wiki/Q2388514","display_name":"Intelligence analysis","level":2,"score":0.36880001425743103},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3644999861717224},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.3319999873638153},{"id":"https://openalex.org/C195344581","wikidata":"https://www.wikidata.org/wiki/Q2555318","display_name":"Automated reasoning","level":2,"score":0.30489999055862427},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2937000095844269},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.29089999198913574},{"id":"https://openalex.org/C105409693","wikidata":"https://www.wikidata.org/wiki/Q5937824","display_name":"Human intelligence","level":2,"score":0.2750999927520752},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.27070000767707825},{"id":"https://openalex.org/C2767350","wikidata":"https://www.wikidata.org/wiki/Q6662173","display_name":"Business intelligence","level":2,"score":0.26089999079704285},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.26010000705718994},{"id":"https://openalex.org/C161301231","wikidata":"https://www.wikidata.org/wiki/Q3478658","display_name":"Knowledge representation and reasoning","level":2,"score":0.2597000002861023},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.25290000438690186},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2515999972820282},{"id":"https://openalex.org/C157170001","wikidata":"https://www.wikidata.org/wiki/Q4781507","display_name":"Applications of artificial intelligence","level":2,"score":0.25099998712539673}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3770855.3817527","doi":"https://doi.org/10.1145/3770855.3817527","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3770855.3817527","pdf_url":null,"source":{"id":"https://openalex.org/S4363608767","display_name":"ACM SIGKDD Conference on Knowledge Discovery and Data Mining (KDD)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2510.11974","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.11974","pdf_url":"https://arxiv.org/pdf/2510.11974","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.11974","doi":"https://doi.org/10.48550/arxiv.2510.11974","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.11974","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1145/3770855.3817527","doi":"https://doi.org/10.1145/3770855.3817527","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3770855.3817527","pdf_url":null,"source":{"id":"https://openalex.org/S4363608767","display_name":"ACM SIGKDD Conference on Knowledge Discovery and Data Mining (KDD)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G6791557944","display_name":"CAREER: Leveraging Large Language Models for Advanced Threat Protection in Cybersecurity","funder_award_id":"2442171","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7204614275","https://openalex.org/W4415797703","https://openalex.org/W7134968946","https://openalex.org/W4416371364","https://openalex.org/W7134929191","https://openalex.org/W7083828117","https://openalex.org/W7081587788","https://openalex.org/W7154268783","https://openalex.org/W4412509665","https://openalex.org/W4414817612"],"abstract_inverted_index":{"Cyber":[0],"Threat":[1],"Intelligence":[2],"(CTI)":[3],"is":[4],"foundational":[5],"to":[6,11,64,85],"modern":[7],"cybersecurity,":[8],"enabling":[9],"organizations":[10],"proactively":[12],"defend":[13],"against":[14],"evolving":[15],"threats.":[16],"However,":[17],"the":[18,41,86,111,152,168,180,216],"sheer":[19],"volume":[20],"and":[21,35,50,141,166,175,219],"heterogeneity":[22],"of":[23,43,52],"CTI":[24,65,112,124,239],"data,":[25],"spanning":[26,131,225],"structured":[27],"knowledge":[28,89],"bases":[29],"(CVE,":[30],"CWE,":[31],"CAPEC,":[32],"MITRE":[33],"ATT&CK)":[34],"unstructured":[36],"threat":[37],"reports,":[38],"far":[39],"exceed":[40],"capacity":[42],"manual":[44],"analysis.":[45],"The":[46],"strong":[47],"contextual":[48],"understanding":[49],"reasoning":[51],"Large":[53],"Language":[54],"Models":[55],"(LLMs)":[56],"have":[57],"driven":[58],"growing":[59],"interest":[60],"in":[61,73,94],"applying":[62],"them":[63],"tasks.":[66],"Yet":[67],"no":[68],"existing":[69],"benchmark":[70,104],"evaluates":[71],"LLMs":[72,109,149],"a":[74,78,103,117],"retrieval-augmented":[75,108],"setting":[76],"with":[77],"proper":[79],"evaluation":[80,119],"harness":[81],"that":[82,151,167,194],"grants":[83],"access":[84],"heterogeneous":[87,123,238],"domain":[88],"sources":[90,125],"analysts":[91],"rely":[92],"on":[93,146,179,215],"practice.":[95],"To":[96],"address":[97],"this":[98,196],"gap,":[99],"we":[100],"present":[101],"CTIConnect,":[102],"for":[105,232],"systematically":[106],"evaluating":[107],"across":[110,134,158,209],"task":[113,159],"landscape.":[114],"We":[115],"construct":[116],"unified":[118],"environment":[120],"integrating":[121],"five":[122],"into":[126],"1,860":[127],"expert-verified":[128],"QA":[129],"pairs":[130],"nine":[132],"tasks":[133],"three":[135],"categories:":[136],"Entity":[137,142],"Linking,":[138],"Multi-Document":[139],"Synthesis,":[140],"Attribution.":[143],"Extensive":[144],"experiments":[145],"ten":[147,211],"state-of-the-art":[148],"reveal":[150],"cross-source":[153],"semantic":[154],"gap":[155,197],"manifests":[156],"differently":[157],"categories,":[160],"demanding":[161],"fundamentally":[162],"different":[163],"retrieval":[164,173,189,204,235],"strategies,":[165],"performance":[169],"bottleneck":[170],"shifts":[171],"between":[172],"infrastructure":[174],"evidence":[176],"utilization":[177],"depending":[178],"task.":[181],"Our":[182],"domain-specific":[183],"strategies":[184],"further":[185],"outperform":[186],"stronger":[187],"general-purpose":[188],"paradigms":[190],"(retrieve-then-rerank,":[191],"IRCoT),":[192],"showing":[193],"closing":[195],"requires":[198],"structural":[199],"interventions":[200],"rather":[201],"than":[202],"generic":[203],"improvements.":[205],"These":[206],"findings":[207],"hold":[208],"all":[210],"LLMs,":[212],"remain":[213],"consistent":[214],"full":[217],"benchmark,":[218],"stay":[220],"stable":[221],"under":[222],"temporal":[223],"splits":[224],"2008-2025.":[226],"Together,":[227],"they":[228],"provide":[229],"actionable":[230],"guidance":[231],"designing":[233],"scalable":[234],"architectures":[236],"over":[237],"ecosystems.":[240]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2025-10-17T00:00:00"}