{"id":"https://openalex.org/W7213758201","doi":"https://doi.org/10.5281/zenodo.22858874","title":"HoneypotBench: A Curated 5-Class MITRE ATT&CK Benchmark for Evaluating Frontier Large Language Models on Honeypot Alert Classification","display_name":"HoneypotBench: A Curated 5-Class MITRE ATT&CK Benchmark for Evaluating Frontier Large Language Models on Honeypot Alert Classification","publication_year":2026,"publication_date":"2026-09-20","ids":{"openalex":"https://openalex.org/W7213758201","doi":"https://doi.org/10.5281/zenodo.22858874"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.22858874","doi":"https://doi.org/10.5281/zenodo.22858874","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.22858874","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},"volume":null,"type":"dataset","is_traditional":false,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.22858874","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133484551","display_name":"Sridhar Govardhan","orcid":"https://orcid.org/0000-0002-2099-9283","observed_orcids":["https://orcid.org/0000-0002-2099-9283"]},"institutions":[{"id":"https://openalex.org/I83737708","display_name":"REVA University","ror":"https://ror.org/03gtcxd54","country_code":"IN","type":"education","lineage":["https://openalex.org/I83737708"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"G, Sridhar","raw_affiliation_strings":["REVA University"],"raw_orcid":"https://orcid.org/0000-0002-2099-9283","affiliations":[{"raw_affiliation_string":"REVA University","institution_ids":["https://openalex.org/I83737708"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5133484551"],"corresponding_institution_ids":["https://openalex.org/I83737708"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10400","display_name":"Network Security and Intrusion Detection","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10400","display_name":"Network Security and Intrusion Detection","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.0010000000474974513,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12326","display_name":"Network Packet Processing and Optimization","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mitre-att-ck","display_name":"MITRE ATT&CK","score":0.8029999732971191},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7599999904632568},{"id":"https://openalex.org/keywords/threat-classification","display_name":"threat classification","score":0.7929999828338623},{"id":"https://openalex.org/keywords/benchmark-dataset","display_name":"benchmark dataset","score":0.5460000038146973},{"id":"https://openalex.org/keywords/multi-annotator-annotation","display_name":"multi-annotator annotation","score":0.6370000243186951},{"id":"https://openalex.org/keywords/sentence-bert","display_name":"Sentence-BERT","score":0.4740000069141388},{"id":"https://openalex.org/keywords/emerging-threats","display_name":"emerging threats","score":0.45899999141693115},{"id":"https://openalex.org/keywords/cyber-threat-intelligence","display_name":"cyber threat intelligence","score":0.4959999918937683},{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.3440000116825104},{"id":"https://openalex.org/keywords/adversarial-prompts","display_name":"adversarial prompts","score":0.3490000069141388},{"id":"https://openalex.org/keywords/prompt-engineering","display_name":"prompt engineering","score":0.34200000762939453},{"id":"https://openalex.org/keywords/adversarial-robustness","display_name":"adversarial robustness","score":0.42399999499320984},{"id":"https://openalex.org/keywords/adversarial-training","display_name":"adversarial training","score":0.3659999966621399}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7900000214576721},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6675999760627747},{"id":"https://openalex.org/C55282118","wikidata":"https://www.wikidata.org/wiki/Q252683","display_name":"Snapshot (computer storage)","level":2,"score":0.6639000177383423},{"id":"https://openalex.org/C191267431","wikidata":"https://www.wikidata.org/wiki/Q911932","display_name":"Honeypot","level":2,"score":0.454800009727478},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.42719998955726624},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.37959998846054077},{"id":"https://openalex.org/C2777683733","wikidata":"https://www.wikidata.org/wiki/Q201456","display_name":"Directory","level":2,"score":0.36550000309944153},{"id":"https://openalex.org/C167981075","wikidata":"https://www.wikidata.org/wiki/Q2667186","display_name":"Sandbox (software development)","level":2,"score":0.3605000078678131},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3546999990940094},{"id":"https://openalex.org/C2778134712","wikidata":"https://www.wikidata.org/wiki/Q1047307","display_name":"Bundle","level":2,"score":0.353300005197525},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3345000147819519},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33070001006126404},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3303999900817871},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.32910001277923584},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.3287000060081482},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.32839998602867126},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.32710000872612},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.31949999928474426},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.3118000030517578},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.30160000920295715},{"id":"https://openalex.org/C2781430460","wikidata":"https://www.wikidata.org/wiki/Q2726043","display_name":"Chromoblastomycosis","level":2,"score":0.2904999852180481},{"id":"https://openalex.org/C171730128","wikidata":"https://www.wikidata.org/wiki/Q5227290","display_name":"Data file","level":2,"score":0.2903999984264374},{"id":"https://openalex.org/C43364308","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Byte","level":2,"score":0.28780001401901245},{"id":"https://openalex.org/C70388272","wikidata":"https://www.wikidata.org/wiki/Q5968558","display_name":"IBM","level":2,"score":0.2808000147342682},{"id":"https://openalex.org/C2780154274","wikidata":"https://www.wikidata.org/wiki/Q7126717","display_name":"Download","level":2,"score":0.2750999927520752},{"id":"https://openalex.org/C2989486834","wikidata":"https://www.wikidata.org/wiki/Q3808900","display_name":"True positive rate","level":2,"score":0.2621000111103058},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.26019999384880066},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.25690001249313354}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.22858874","doi":"https://doi.org/10.5281/zenodo.22858874","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.22858874","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},{"id":"doi:10.5281/zenodo.22858875","doi":"https://doi.org/10.5281/zenodo.22858875","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.22858875","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"}],"best_oa_location":{"id":"doi:10.5281/zenodo.22858874","doi":"https://doi.org/10.5281/zenodo.22858874","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.22858874","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7165477255","https://openalex.org/W7203649506","https://openalex.org/W7203617937","https://openalex.org/W7117235239","https://openalex.org/W7155160457","https://openalex.org/W7107858491","https://openalex.org/W7171403511","https://openalex.org/W7212059063","https://openalex.org/W4414404744","https://openalex.org/W7214625388"],"abstract_inverted_index":{"#":[0],"HoneypotBench":[1,87,187,332,749],"\u2014":[2,378,600,614,625,636,766,909,1022,1215],"Complete":[3],"Data":[4,41],"and":[5,89,136,675,702,791,842,929,1067,1083,1102,1107,1114,1154,1204,1217],"Code":[6,48],"Release":[7],"(v1.0)":[8],"**Companion":[9],"artefact":[10,83],"to:**":[11],"*HoneypotBench:":[12],"A":[13,80,96],"Curated":[14],"5-Class":[15],"MITRE":[16,477,986,1180],"ATT&CK":[17,478],"Benchmark":[18],"for":[19,85,310,584,794,807,847,914,965,1128,1159],"Evaluating":[20],"Frontier":[21],"Large":[22],"Language":[23],"Models":[24],"on":[25,71,119,1099],"Cloud-Honeypot":[26],"Threat":[27],"Classification.*":[28],"**Authors:**":[29],"Sridhar":[30],"G,":[31],"J":[32],"B":[33],"Simha,":[34],"Rashmi":[35],"Agarwal":[36],"(REVA":[37],"University,":[38],"Bengaluru)":[39],"**Licence:**":[40],"files":[42,149],"(`.csv`,":[43],"`.json`,":[44],"`.jsonl`)":[45],"under":[46,50,57,1092],"**CC-BY-4.0**.":[47],"(`.py`)":[49],"**MIT**.":[51],"Third-party":[52],"rule":[53,1095],"snapshot":[54,165],"(`emerging_rules_snapshot.tar.gz`)":[55],"redistributed":[56],"its":[58],"original":[59],"BSD-style":[60],"Emerging":[61],"Threats":[62],"Open":[63],"license.":[64],"**Cite":[65],"as:**":[66],"[DOI":[67],"to":[68,132,167,282,447,748,773,777,1169],"be":[69],"assigned":[70],"Zenodo":[72],"publication]":[73],"---":[74,173,322],"##":[75,174,323],"What":[76],"this":[77,180,690],"release":[78],"contains":[79],"complete,":[81],"reproducible":[82],"bundle":[84],"the":[86,94,101,110,116,129,139,154,163,169,299,336,347,352,380,496,679,733,738,758,763,771,784,808,851,910,917,946,966,1004,1015,1019,1023,1034,1064,1093,1108,1129,1170,1173,1201,1205],"benchmark":[88,102],"every":[90,106],"analysis":[91,1036],"reported":[92,107,170,311],"in":[93,123,160,343,468,567,686,722,762,850,951,1007,1047,1132],"manuscript.":[95,348],"reader":[97],"can":[98,1062],"(a)":[99],"inspect":[100,1063],"itself,":[103],"(b)":[104],"reproduce":[105,1068],"metric":[108,685],"from":[109,689,700,732,972,999,1033],"raw":[111,195,1065],"per-alert":[112,203,210,576],"predictions,":[113,1153],"(c)":[114],"re-run":[115],"classical":[117],"baselines":[118],"their":[120],"own":[121],"machine":[122],"about":[124],"15":[125],"minutes,":[126],"(d)":[127],"extend":[128],"LLM":[130,196,216,577,801],"evaluation":[131],"future":[133],"frontier":[134,587,810],"models,":[135],"(e)":[137],"audit":[138],"multi-annotator":[140],"hallucination":[141,967,1175],"adjudication.":[142],"**All":[143],"primary":[144,337,601],"artefacts":[145],"are":[146],"included.**":[147],"No":[148],"require":[150],"external":[151,1160],"download":[152],"except":[153],"ETOpen":[155,307,741],"ruleset,":[156],"which":[157],"is":[158,335,341,641,792,988,1012],"bundled":[159],"`rules/`":[161],"as":[162,382],"exact":[164],"used":[166,309,964,1127],"produce":[168],"Suricata":[171,209,231,734,942],"baseline.":[172],"Directory":[175],"layout":[176],"```HoneypotBench_v1/\u251c\u2500\u2500":[177],"README.md":[178],"\u2190":[179,185,193,200,207,215,223,230,238,244,250,258,264,272,279,287,292,297,305],"file\u251c\u2500\u2500":[181],"data/\u2502":[182],"\u2514\u2500\u2500":[183,205,228,277,295,303,320],"dataset.csv":[184],"1,947-alert":[186,331],"(primary":[188],"artefact)\u251c\u2500\u2500":[189],"predictions/\u2502":[190],"\u251c\u2500\u2500":[191,198,213,221,236,242,248,256,262,270,285,290,314,316,318],"llm_predictions.jsonl":[192],"7,641":[194,573],"predictions\u2502":[197,204],"sbert_baseline_predictions.jsonl":[199],"1,947":[201,208,598,696,728],"SBERT":[202],"suricata_predictions.jsonl":[206],"predictions\u251c\u2500\u2500":[211],"baselines/\u2502":[212],"llm_results.json":[214],"aggregate":[217,684,854],"results":[218,802,908],"(all":[219],"metrics)\u2502":[220],"sbert_baseline_metrics.json":[222],"SBERT+XGBoost":[224,701,915,1216],"+":[225],"SBERT+RF":[226],"results\u2502":[227],"suricata_baseline_metrics.json":[229],"rule-content":[232,735],"matcher":[233,736],"results\u251c\u2500\u2500":[234],"adjudication/\u2502":[235],"hallucination_adjudication_stratified_sample.csv":[237],"211-alert":[239],"base":[240,961],"sample\u2502":[241],"annotator_1_verdicts.csv":[243],"Annotator":[245,251,259,1028,1054],"1":[246,345,1082],"verdicts\u2502":[247,261],"annotator_2_verdicts_excluded.csv":[249],"2":[252],"(excluded,":[253],"see":[254],"\u00a76.9)\u2502":[255],"annotator_3_verdicts.csv":[257],"3":[260,1055,1084],"hallucination_adjudication_consensus_A1_A3.csv":[263],"Merged":[265,1078],"A1+A3":[266],"with":[267,645,668,815,820,828,1014,1080],"strict":[268,1094],"consensus\u2502":[269],"expert_adjudication_set_657rows.csv":[271],"Section":[273,615,626,637],"7":[274,409],"expert-adjudication":[275],"export\u2502":[276],"Annotator_Brief.md":[278],"Brief":[280],"distributed":[281,1168],"annotators\u251c\u2500\u2500":[283],"code/\u2502":[284],"run_sbert_baselines.py":[286],"Reproduces":[288,293,945],"sbert_baseline_metrics.json\u2502":[289],"run_suricata_baseline.py":[291],"suricata_baseline_metrics.json\u2502":[294],"hallucination_adjudication_export.py":[296],"Regenerates":[298],"stratified":[300,506,518,612,708,962,1125,1221],"sample\u251c\u2500\u2500":[301],"rules/\u2502":[302],"emerging_rules_snapshot.tar.gz":[304],"Exact":[306,1166],"ruleset":[308,742],"Suricata\u2514\u2500\u2500":[312],"figures/":[313],"fig_calibration.png":[315],"fig_confusion.png":[317],"fig_cost_accuracy.png":[319],"fig_perclass_f1.png```":[321],"File-by-file":[324],"description":[325],"###":[326,569,692,724,796,901,938,954,990,1074,1119,1162,1208],"`data/dataset.csv`":[327],"(573":[328],"KB)":[329,799,904,941,1077,1122,1165],"The":[330,339,960,1123],"corpus.":[333],"**This":[334],"artefact.**":[338],"schema":[340],"described":[342],"Table":[344,1105],"of":[346,495,1142],"Every":[349,683],"alert":[350],"carries":[351],"following":[353],"columns:":[354],"|":[355,357,359,363,365,373,375,386,388,399,402,415,417,423,425,430,432,438,440,451,453,463,465,473,475,482,484,491,493,501,503,513,515,523,525,531,858,860,862,864,870,872,874,882,884,886,893,895,897,900],"Column":[356],"Type":[358,1182,1187,1192,1197],"Description":[360],"||---|---|---||":[361],"`alert_id`":[362],"string":[364,424,431,439,474,483,492],"Unique":[366],"identifier:":[367],"`TRI-0001`":[368],"through":[369],"`TRI-1947`":[370],"||":[371,384,396,412,421,428,436,449,461,471,480,489,499,511,521,877,889],"`synthetic`":[372],"bool":[374],"Always":[376],"`True`":[377],"flags":[379],"record":[381],"benchmark-generated":[383],"`category`":[385],"enum":[387,452,502],"Ground-truth":[389],"class:":[390],"`brute_force`,":[391],"`c2_malware`,":[392,778],"`web_exploitation`,":[393],"`reconnaissance`,":[394],"`benign_noise`":[395],"`first_seen`,":[397],"`last_seen`":[398],"ISO":[400],"8601":[401],"Time":[403],"span":[404],"(within":[405],"18":[406],"Oct":[407],"\u2013":[408],"Nov":[410],"2025)":[411],"`event_count`,":[413],"`distinct_paths`":[414],"int":[416,514,524],"Session-level":[418],"activity":[419],"summary":[420],"`target_service`":[422],"Application-layer":[426],"service":[427,486],"`target_honeypot`":[429],"Deployed":[433],"honeypot":[434,565],"component":[435],"`source_ip`":[437],"RFC":[441],"5737":[442],"documentation":[443],"IPv4":[444],"address":[445],"(safe":[446],"redistribute)":[448],"`severity`":[450],"`CRITICAL`":[454],"/":[455,457,459,509,650,652,656,658,660,1156],"`HIGH`":[456],"`MEDIUM`":[458],"`INFO`":[460],"`confidence`":[462],"float":[464],"Template-assigned":[466],"confidence":[467],"[0,":[469],"1]":[470],"`mitre_techniques`":[472],"Semicolon-separated":[476],"IDs":[479],"`evidence_sample`":[481],"Realistic":[485],"log":[487],"excerpt":[488],"`template_id`":[490],"Identifier":[494],"generation":[497],"template":[498],"`holdout_70_30`":[500],"Pre-computed":[504,516],"70/30":[505],"split:":[507],"`train`":[508],"`test`":[510],"`cv_fold_5`":[512],"5-fold":[517,527,707,1220],"assignment":[519],"(0\u20134)":[520],"`grouped_fold_5`":[522],"Alternative":[526],"grouped":[528],"by":[529,706],"`source_ip_hash`":[530],"**Class":[532],"balance:**":[533],"390":[534,536],"brute_force,":[535],"c2_malware,":[537],"389":[538,540,542],"web_exploitation,":[539],"reconnaissance,":[541],"benign_noise.":[543],"**Service":[544],"distribution:**":[545],"webtrap":[546],"860,":[547],"snare":[548],"443,":[549],"mysql":[550],"295,":[551],"jupyter":[552],"123,":[553],"cowrie":[554],"107,":[555],"smtp":[556],"50,":[557],"k8s":[558],"41,":[559],"redis":[560],"20,":[561],"conpot":[562],"8.":[563],"**Nine":[564],"services":[566],"total.**":[568],"`predictions/llm_predictions.jsonl`":[570],"(4.4":[571],"MB,":[572],"records)":[574,697,729],"Raw":[575],"predictions":[578,699,731,776],"across":[579,803],"all":[580,585,775,804,1040,1046,1150],"four":[581,805,973],"experimental":[582],"conditions":[583],"three":[586,809,1000,1151],"models.":[588],"Breakdown:":[589],"-":[590,1213],"**Baseline":[591],"evaluation:**":[592],"5,841":[593],"records":[594,608,621,632],"(3":[595,609,622,633],"models":[596],"\u00d7":[597,610,623,634],"alerts)":[599],"evaluation,":[602],"main-text":[603],"tables-":[604],"**Few-shot":[605],"ablation:**":[606],"1,500":[607],"500-alert":[611],"subset)":[613],"6.4-":[616],"**Naive":[617],"adversarial":[618,629],"suffix:**":[619,630],"150":[620,631],"50)":[624,635],"6.5-":[627],"**Sophisticated":[628],"6.5":[638],"Each":[639,710,751],"line":[640,711],"a":[642,1088],"JSON":[643,666],"object":[644],"fields:":[646],"`alert_id`,":[647,713,753],"`model_key`":[648],"(`claude`":[649],"`gpt`":[651],"`gemini`),":[653],"`tag`":[654],"(`baseline`":[655],"`fewshot`":[657],"`adv_naive`":[659],"`adv_sophisticated`),":[661],"`parsed`":[662],"(the":[663],"model's":[664],"parsed":[665],"response":[667],"`classification`,":[669],"`attack_category`,":[670],"`mitre_attack_technique_id`,":[671],"`severity`,":[672],"`confidence`),":[673],"`raw_response`,":[674],"cost":[676,838],"fields":[677],"where":[678],"API":[680],"exposes":[681],"them.":[682],"`llm_results.json`":[687],"derives":[688],"file.":[691],"`predictions/sbert_baseline_predictions.jsonl`":[693],"(389":[694],"KB,":[695,727,957],"Per-alert":[698,730],"SBERT+Random":[703],"Forest,":[704],"produced":[705],"cross-validation.":[709],"has:":[712],"`ground_truth`,":[714,754],"`fold`,":[715],"`sbert_xgb_pred`,":[716],"`sbert_xgb_confidence`,":[717],"`sbert_rf_pred`,":[718],"`sbert_rf_confidence`.":[719],"Aggregate":[720,800],"metrics":[721],"`sbert_baseline_metrics.json`.":[723],"`predictions/suricata_predictions.jsonl`":[725],"(312":[726],"against":[737,932,1053],"December":[739],"2025":[740],"(52,160":[743],"rules":[744,769],"parsed,":[745],"37,352":[746],"mapped":[747],"categories).":[750],"line:":[752],"`suricata_pred`,":[755],"`matched_rule_msg`.":[756],"Note":[757],"degenerate":[759],"behaviour":[760],"discussed":[761,950],"manuscript":[764,918,1024],"\u00a79":[765,952],"broad":[767],"C2":[768],"cause":[770],"classifier":[772],"collapse":[774],"yielding":[779],"0.20":[780,947],"accuracy.":[781],"This":[782],"measures":[783],"rule-selection":[785],"protocol,":[786],"not":[787],"Suricata's":[788],"operational":[789],"capability,":[790],"included":[793,1013],"reproducibility.":[795],"`baselines/llm_results.json`":[797],"(65":[798],"tags":[806],"LLMs.":[811],"Contains":[812,921,1145],"per-model:":[813],"accuracy":[814,913,948],"95%":[816,865,927],"bootstrap":[817,926],"CIs,":[818,821,928],"macro-F1":[819],"per-class":[822,922],"metrics,":[823],"confusion":[824,924],"matrices,":[825,925],"calibration":[826],"bins":[827],"ECE,":[829],"per-service":[830],"breakdown,":[831],"ensemble":[832],"analysis,":[833],"prompt-sensitivity":[834],"deltas,":[835,837],"adversarial-robustness":[836],"breakdown":[839],"per":[840,1018,1146],"tag,":[841],"reasoning-token":[843],"counts.":[844],"**Authoritative":[845],"source":[846],"Tables":[848],"4\u201314":[849],"manuscript.**":[852],"Baseline":[853],"(for":[855],"quick":[856],"verification):":[857],"Model":[859],"Accuracy":[861],"Macro-F1":[863],"CI":[866],"(accuracy)":[867],"||---|---|---|---||":[868],"GPT-5.6-sol":[869,933],"0.5727":[871],"0.5161":[873],"[0.5501,":[875],"0.5948]":[876],"Gemini":[878],"3.1":[879],"Pro":[880],"Preview":[881],"0.5280":[883],"0.4633":[885],"[0.5054,":[887],"0.5501]":[888],"Claude":[890],"Fable":[891],"5.1":[892],"0.4643":[894],"0.4335":[896],"[0.4412,":[898],"0.4864]":[899],"`baselines/sbert_baseline_metrics.json`":[902],"(4":[903],"Classical":[905],"ML":[906],"baseline":[907,943],"surprising":[911],"0.957":[912],"that":[916,1143],"\u00a76.7":[919],"discusses.":[920],"precision/recall/F1,":[923],"McNemar":[930],"tests":[931],"(b=36,":[934],"c=785,":[935],"\u03c7\u00b2=681.5,":[936],"p<10\u207b\u00b9\u2075\u2070).":[937],"`baselines/suricata_baseline_metrics.json`":[939],"(2":[940],"metrics.":[944],"figure":[949],"Limitations.":[953],"`adjudication/hallucination_adjudication_stratified_sample.csv`":[955],"(60":[956],"211":[958,995],"rows)":[959],"sample":[963],"adjudication":[968,1131],"(\u00a76.9).":[969],"Alerts":[970],"drawn":[971],"buckets:":[974],"confident-and-wrong":[975],"(30%":[976],"target),":[977],"confident-and-right":[978],"(30%),":[979],"low-confidence":[980],"(20%),":[981],"unusual-MITRE":[982],"(20%).":[983],"Type-1":[984],"(structural":[985],"format)":[987],"pre-flagged.":[989],"`adjudication/annotator_{1,2,3}_verdicts.csv`":[991],"(61\u201366":[992],"KB":[993],"each,":[994],"rows":[996,1043],"each)":[997],"Verdicts":[998],"security-domain":[1001],"annotators":[1002,1097],"using":[1003],"five-type":[1005],"rubric":[1006],"`Annotator_Brief.md`.":[1008],"**Annotator":[1009],"2's":[1010,1029],"file":[1011,1079],"`_excluded`":[1016],"suffix":[1017],"openness":[1020],"principle**":[1021],"\u00a76.9":[1025],"discusses":[1026],"why":[1027],"verdicts":[1030,1066,1085],"were":[1031,1044],"excluded":[1032],"headline":[1035,1117,1140],"(systematic":[1037],"labelling":[1038],"artefact:":[1039],"six":[1041],"flagged":[1042],"type_2,":[1045],"Claude,":[1048],"no":[1049],"notes":[1050],"provided;":[1051],"\u03ba":[1052,1110],"was":[1056],"essentially":[1057],"zero":[1058],"at":[1059],"\u22120.019).":[1060],"Readers":[1061],"or":[1069],"challenge":[1070],"our":[1071],"exclusion":[1072],"decision.":[1073],"`adjudication/hallucination_adjudication_consensus_A1_A3.csv`":[1075],"(73":[1076],"Annotators":[1081],"side-by-side,":[1086],"plus":[1087],"computed":[1089],"`consensus_verdict`":[1090],"column":[1091],"(both":[1096],"agree":[1098],"both":[1100],"flag":[1101],"type).":[1103],"Supports":[1104],"16":[1106],"Cohen's":[1109],"=":[1111],"0.08":[1112],"(5-way)":[1113],"0.18":[1115],"(binary)":[1116],"numbers.":[1118],"`adjudication/expert_adjudication_set_657rows.csv`":[1120],"(261":[1121],"657-alert":[1124],"export":[1126],"expert":[1130],"\u00a77":[1133],"(which":[1134],"found":[1135],"486":[1136],"unanimous":[1137],"LLM-vs-benchmark":[1138],"disagreements,":[1139],"finding":[1141],"section).":[1144],"alert:":[1147],"ground-truth":[1148],"label,":[1149],"models'":[1152],"`expert_verdict`":[1155],"`notes`":[1157],"columns":[1158],"reviewers.":[1161],"`adjudication/Annotator_Brief.md`":[1163],"(7":[1164],"brief":[1167],"annotators.":[1171],"Documents":[1172],"5-type":[1174],"taxonomy":[1176],"(Type":[1177],"1:":[1178],"fabricated":[1179],"ID;":[1181],"2:":[1183],"phantom":[1184],"entity":[1185],"attribution;":[1186],"3:":[1188],"unsupported":[1189],"severity":[1190],"inflation;":[1191],"4:":[1193],"wrong":[1194],"kill-chain":[1195],"phase;":[1196],"5:":[1198],"temporal/contextual":[1199],"confabulation),":[1200],"review":[1202],"workflow,":[1203],"confidentiality":[1206],"terms.":[1207],"`code/`":[1209],"(three":[1210],"Python":[1211],"scripts)":[1212],"**`run_sbert_baselines.py`**":[1214],"SBERT+RandomForest":[1218],"baselines,":[1219],"CV,":[1222],"bootst":[1223]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-21T00:00:00"}