{"id":"https://openalex.org/W7125899683","doi":"https://doi.org/10.1109/smc58881.2025.11343068","title":"Robustness Evaluation of Tactics, Techniques, and Procedures Knowledge in Large Language Models","display_name":"Robustness Evaluation of Tactics, Techniques, and Procedures Knowledge in Large Language Models","publication_year":2025,"publication_date":"2025-10-05","ids":{"openalex":"https://openalex.org/W7125899683","doi":"https://doi.org/10.1109/smc58881.2025.11343068"},"language":null,"primary_location":{"id":"doi:10.1109/smc58881.2025.11343068","is_oa":false,"landing_page_url":"https://doi.org/10.1109/smc58881.2025.11343068","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Systems, Man, and Cybernetics (SMC)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008281260","display_name":"Yikai Chen","orcid":"https://orcid.org/0000-0002-3706-2569"},"institutions":[{"id":"https://openalex.org/I4391767781","display_name":"State Key Laboratory of Software Development Environment","ror":"https://ror.org/017r4dj03","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391767781","https://openalex.org/I82880672"]},{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yikai Chen","raw_affiliation_strings":["Beihang University,State Key Laboratory of Software Development Environment,Beijing,China,100191"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beihang University,State Key Laboratory of Software Development Environment,Beijing,China,100191","institution_ids":["https://openalex.org/I4391767781","https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086522441","display_name":"Bo Lang","orcid":"https://orcid.org/0000-0002-0586-3783"},"institutions":[{"id":"https://openalex.org/I4391767781","display_name":"State Key Laboratory of Software Development Environment","ror":"https://ror.org/017r4dj03","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391767781","https://openalex.org/I82880672"]},{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bo Lang","raw_affiliation_strings":["Beihang University,State Key Laboratory of Software Development Environment,Beijing,China,100191"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beihang University,State Key Laboratory of Software Development Environment,Beijing,China,100191","institution_ids":["https://openalex.org/I4391767781","https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101942275","display_name":"Nan Xiao","orcid":"https://orcid.org/0000-0002-6487-9881"},"institutions":[{"id":"https://openalex.org/I4391767781","display_name":"State Key Laboratory of Software Development Environment","ror":"https://ror.org/017r4dj03","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391767781","https://openalex.org/I82880672"]},{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Nan Xiao","raw_affiliation_strings":["Beihang University,State Key Laboratory of Software Development Environment,Beijing,China,100191"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beihang University,State Key Laboratory of Software Development Environment,Beijing,China,100191","institution_ids":["https://openalex.org/I4391767781","https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100460308","display_name":"Xiangyu Li","orcid":"https://orcid.org/0000-0001-9903-9797"},"institutions":[{"id":"https://openalex.org/I4391767781","display_name":"State Key Laboratory of Software Development Environment","ror":"https://ror.org/017r4dj03","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391767781","https://openalex.org/I82880672"]},{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiangyu Li","raw_affiliation_strings":["Beihang University,State Key Laboratory of Software Development Environment,Beijing,China,100191"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beihang University,State Key Laboratory of Software Development Environment,Beijing,China,100191","institution_ids":["https://openalex.org/I4391767781","https://openalex.org/I82880672"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101770902","display_name":"Ruibo Chen","orcid":"https://orcid.org/0009-0009-9587-3798"},"institutions":[{"id":"https://openalex.org/I4391767781","display_name":"State Key Laboratory of Software Development Environment","ror":"https://ror.org/017r4dj03","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391767781","https://openalex.org/I82880672"]},{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruibo Chen","raw_affiliation_strings":["Beihang University,State Key Laboratory of Software Development Environment,Beijing,China,100191"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beihang University,State Key Laboratory of Software Development Environment,Beijing,China,100191","institution_ids":["https://openalex.org/I4391767781","https://openalex.org/I82880672"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.65413273,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4429","last_page":"4436"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.12700000405311584,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.12700000405311584,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12519","display_name":"Cybercrime and Law Enforcement Studies","score":0.10360000282526016,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0649000033736229,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.8629000186920166},{"id":"https://openalex.org/keywords/evaluation-methods","display_name":"Evaluation methods","score":0.33379998803138733},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.2590000033378601},{"id":"https://openalex.org/keywords/ranging","display_name":"Ranging","score":0.2565000057220459},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.24699999392032623}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.8629000186920166},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6991000175476074},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.460099995136261},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45890000462532043},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4068000018596649},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.33379998803138733},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.2590000033378601},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.2565000057220459},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.24699999392032623},{"id":"https://openalex.org/C56814567","wikidata":"https://www.wikidata.org/wiki/Q1323686","display_name":"Explicit knowledge","level":2,"score":0.23569999635219574}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/smc58881.2025.11343068","is_oa":false,"landing_page_url":"https://doi.org/10.1109/smc58881.2025.11343068","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Systems, Man, and Cybernetics (SMC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5529412031173706,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W2771963642","https://openalex.org/W2964522977","https://openalex.org/W2970476646","https://openalex.org/W3199212845","https://openalex.org/W4210404000","https://openalex.org/W4382318164","https://openalex.org/W4385571596","https://openalex.org/W4385572734","https://openalex.org/W4392947753","https://openalex.org/W4402670248","https://openalex.org/W4404515031","https://openalex.org/W4404534210","https://openalex.org/W4404783224","https://openalex.org/W4408566170","https://openalex.org/W4409656997","https://openalex.org/W4415797703","https://openalex.org/W7120088840"],"related_works":[],"abstract_inverted_index":{"Extracting":[0],"Tactics,":[1],"Techniques,":[2],"and":[3,33,58,80,114,126,173,231],"Procedures":[4],"(TTP)":[5],"from":[6,98,178],"unstructured":[7],"threat":[8,16],"reports":[9],"remains":[10],"a":[11,50,90,99,106,186],"critical":[12,224],"challenge":[13],"in":[14,61],"cyber":[15],"intelligence":[17],"(CTI).":[18],"Although":[19],"large":[20],"language":[21],"models":[22,132,149,201],"(LLMs)":[23],"may":[24],"automate":[25],"TTP":[26,31,56,229],"extraction,":[27],"their":[28,140,210],"understanding":[29,135],"of":[30,41,77,83,119,136,159,169],"knowledge":[32,57,91,101],"robustness":[34,59,141,184,207],"remain":[35],"unverified":[36],"due":[37],"to":[38,94,166,180,209],"the":[39,55,75,81,111,122,157,167,194,199],"lack":[40],"evaluation":[42,118,195,219],"benchmarks.":[43],"To":[44],"address":[45],"this,":[46],"we":[47,70,88,104],"propose":[48],"TTP-RoB,":[49,64],"novel":[51],"framework":[52],"for":[53,220,228],"probing":[54,107],"inherent":[60],"LLMs.":[62],"In":[63],"by":[65],"analyzing":[66],"real-world":[67],"CTI":[68,233],"reports,":[69],"identify":[71],"key":[72],"interference":[73,82,146,168],"factors:":[74],"absence":[76,158],"correct":[78,160],"answers":[79],"similar":[84,170],"technique":[85,171],"names.":[86],"Then,":[87],"develop":[89],"sampling":[92],"algorithm":[93],"select":[95],"representative":[96],"examples":[97],"relevant":[100],"base.":[102],"Finally,":[103],"construct":[105],"dataset":[108],"that":[109,129,198,217],"integrates":[110],"identified":[112],"interferences":[113],"sampled":[115],"knowledge.Our":[116],"TTP-RoB":[117],"mainstream":[120],"LLMs\u2014including":[121],"Deepseek,":[123],"GPT,":[124],"Llama,":[125],"Qwen":[127],"series\u2014demonstrate":[128],"while":[130],"all":[131],"show":[133],"strong":[134],"basic":[137],"technical":[138],"concepts,":[139],"varies":[142],"significantly":[143,205],"under":[144],"different":[145],"conditions.":[147],"The":[148],"generally":[150],"maintain":[151],"stable":[152],"performance":[153],"when":[154],"faced":[155],"with":[156],"answers,":[161],"but":[162],"display":[163],"substantial":[164],"vulnerability":[165],"names":[172],"suffer":[174],"accuracy":[175],"drops":[176],"ranging":[177],"15.4%":[179],"59.1%.":[181],"Overall,":[182],"model":[183,190,221],"exhibits":[185],"significant":[187],"decline":[188],"as":[189],"scale":[191],"decreases.":[192],"Furthermore,":[193],"results":[196,215],"reveal":[197],"distilled":[200],"(e.g.,":[202],"DeepSeek-R1-distill-Qwen)":[203],"exhibit":[204],"lower":[206],"compared":[208],"corresponding":[211],"foundation":[212],"models.":[213],"These":[214],"confirm":[216],"rigorous":[218],"selection":[222],"is":[223],"before":[225],"using":[226],"LLMs":[227],"extraction":[230],"other":[232],"tasks.":[234]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-01-29T00:00:00"}