{"id":"https://openalex.org/W7164994913","doi":"https://doi.org/10.48550/arxiv.2606.18166","title":"Evaluating Open-Source LLMs for Multi-Label ATT&CK Technique Classification on CTI Reports","display_name":"Evaluating Open-Source LLMs for Multi-Label ATT&CK Technique Classification on CTI Reports","publication_year":2026,"publication_date":"2026-06-16","ids":{"openalex":"https://openalex.org/W7164994913","doi":"https://doi.org/10.48550/arxiv.2606.18166"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.18166","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18166","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.18166","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138763906","display_name":"Ahmed Ryan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ryan, Ahmed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109171840","display_name":"S. S. M. Noor","orcid":"https://orcid.org/0009-0002-7936-6867"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Noor, Saad Sakib","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138812357","display_name":"Md Erfan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Erfan, Md","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075202853","display_name":"Shaswata Mitra","orcid":"https://orcid.org/0009-0002-9722-5312"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mitra, Shaswata","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074949168","display_name":"Sudip Mittal","orcid":"https://orcid.org/0000-0001-9151-8347"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mittal, Sudip","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133347304","display_name":"Md Rayhanur Rahman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rahman, Md Rayhanur","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.42570000886917114,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.42570000886917114,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.09570000320672989,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12519","display_name":"Cybercrime and Law Enforcement Studies","score":0.07020000368356705,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.532800018787384},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.5177000164985657},{"id":"https://openalex.org/keywords/unstructured-data","display_name":"Unstructured data","score":0.48339998722076416},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.34790000319480896},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.335999995470047},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.3125999867916107}],"concepts":[{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.532800018787384},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.5177000164985657},{"id":"https://openalex.org/C2781252014","wikidata":"https://www.wikidata.org/wiki/Q1141900","display_name":"Unstructured data","level":3,"score":0.48339998722076416},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47839999198913574},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4722000062465668},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4120999872684479},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40070000290870667},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.34790000319480896},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.335999995470047},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3133000135421753},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.3125999867916107},{"id":"https://openalex.org/C2776230583","wikidata":"https://www.wikidata.org/wiki/Q1322198","display_name":"Spoken language","level":2,"score":0.27639999985694885},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2685999870300293},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.2669000029563904},{"id":"https://openalex.org/C148524875","wikidata":"https://www.wikidata.org/wiki/Q6975395","display_name":"F1 score","level":2,"score":0.26170000433921814},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.25699999928474426},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.25589999556541443}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.18166","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18166","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.18166","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18166","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.5559540390968323,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Classifying":[0],"Cyber":[1],"Threat":[2],"Intelligence":[3],"(CTI)":[4],"using":[5,55,131],"MITRE":[6],"Adversarial":[7],"Tactics,":[8],"Techniques,":[9],"and":[10,41,159,197,223],"Common":[11],"Knowledge":[12],"(ATT&CK)":[13],"is":[14],"essential":[15],"for":[16,176,216,229],"proactive":[17],"defense,":[18],"but":[19,34],"historically":[20],"required":[21],"extensive":[22],"human":[23],"effort.":[24],"Pre-Large":[25],"Language":[26],"Model":[27],"(LLM)":[28],"automation":[29],"sped":[30],"up":[31],"this":[32,101,143],"process,":[33,135],"could":[35],"not":[36],"resolve":[37],"the":[38,72,86,173],"complex":[39,93,118,181],"language":[40],"multi-step":[42],"attack":[43],"patterns":[44],"found":[45],"in":[46],"unstructured":[47,60,94,119,182],"CTI":[48,76,95,120,231],"reports.":[49,121],"LLMs":[50,91,149,213],"addressed":[51],"previous":[52],"limitations":[53],"by":[54],"contextual":[56],"reasoning":[57],"to":[58,81,126,153],"understand":[59],"text.":[61],"However,":[62],"current":[63,211],"evaluations":[64],"rely":[65],"on":[66,92,180],"simplified,":[67],"single-technique":[68],"sentences":[69,111,123],"that":[70,210],"ignore":[71],"complexity":[73],"of":[74,89,108,170],"real-world":[75],"reports,":[77],"which":[78],"often":[79],"leads":[80],"inflated":[82],"performance":[83,88],"results.":[84],"Consequently,":[85],"baseline":[87,175],"open-source":[90,148,212],"reports":[96],"remains":[97],"unevaluated.":[98],"To":[99],"address":[100],"gap,":[102],"we":[103,145],"constructed":[104],"a":[105,132,166,187,226],"ground-truth":[106],"dataset":[107],"2,076":[109],"human-annotated":[110],"(1,281":[112],"technique-positive,":[113],"795":[114],"negative)":[115],"from":[116,151],"83":[117],"These":[122,207],"were":[124],"mapped":[125],"114":[127],"unique":[128],"ATT&CK":[129,178,218],"techniques":[130],"six-phase":[133],"annotation":[134],"achieving":[136],"\\k{appa}":[137],"=":[138],"0.68":[139],"inter-annotator":[140],"agreement.":[141],"Using":[142],"dataset,":[144,221],"evaluated":[146],"seven":[147],"ranging":[150],"8B":[152],"236B":[154],"parameters":[155],"across":[156,204],"prompt":[157],"strategy":[158,196],"temperature":[160,198],"configurations.":[161,206],"The":[162,220],"highest-performing":[163],"LLM":[164],"achieved":[165],"micro-averaged":[167],"F1":[168,193],"score":[169],"0.22,":[171],"establishing":[172],"empirical":[174],"multi-label":[177],"classification":[179],"CTI.":[183],"Parameter":[184],"size":[185],"showed":[186],"statistically":[188,201],"significant":[189,202],"positive":[190],"correlation":[191],"with":[192],"score.":[194],"Prompt":[195],"produced":[199],"no":[200],"gains":[203],"model":[205],"results":[208],"indicate":[209],"are":[214],"insufficient":[215],"production-grade":[217],"classification.":[219],"benchmark,":[222],"findings":[224],"provide":[225],"reproducible":[227],"foundation":[228],"future":[230],"research.":[232]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-18T00:00:00"}