{"id":"https://openalex.org/W7165383084","doi":"https://doi.org/10.1145/3765611.3815141","title":"CritBench: A Framework for Evaluating Cybersecurity Capabilities of Large Language Models in IEC 61850 Digital Substation Environments","display_name":"CritBench: A Framework for Evaluating Cybersecurity Capabilities of Large Language Models in IEC 61850 Digital Substation Environments","publication_year":2026,"publication_date":"2026-06-20","ids":{"openalex":"https://openalex.org/W7165383084","doi":"https://doi.org/10.1145/3765611.3815141"},"language":null,"primary_location":{"id":"doi:10.1145/3765611.3815141","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3765611.3815141","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Sustainability Week","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3765611.3815141","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100022307","display_name":"Gustav Keppler","orcid":"https://orcid.org/0000-0002-2323-0533"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Gustav Keppler","raw_affiliation_strings":["Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0000-0002-2323-0533","affiliations":[{"raw_affiliation_string":"Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098085533","display_name":"Moritz Gst\u00fcr","orcid":"https://orcid.org/0009-0002-4036-9032"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Moritz Gst\u00fcr","raw_affiliation_strings":["Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0009-0002-4036-9032","affiliations":[{"raw_affiliation_string":"Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014228448","display_name":"Veit Hagenmeyer","orcid":"https://orcid.org/0000-0002-3572-9083"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Veit Hagenmeyer","raw_affiliation_strings":["Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0000-0002-3572-9083","affiliations":[{"raw_affiliation_string":"Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I102335020"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"55","last_page":"64"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10917","display_name":"Smart Grid Security and Resilience","score":0.5942000150680542,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10917","display_name":"Smart Grid Security and Resilience","score":0.5942000150680542,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.09889999777078629,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.024299999698996544,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/iec-61850","display_name":"IEC 61850","score":0.6590999960899353},{"id":"https://openalex.org/keywords/modeling-language","display_name":"Modeling language","score":0.35440000891685486},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.3488999903202057},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.3131999969482422},{"id":"https://openalex.org/keywords/information-model","display_name":"Information model","score":0.311599999666214},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.2736999988555908}],"concepts":[{"id":"https://openalex.org/C2778907243","wikidata":"https://www.wikidata.org/wiki/Q168160","display_name":"IEC 61850","level":3,"score":0.6590999960899353},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5753999948501587},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.35440000891685486},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3488999903202057},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3384999930858612},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.3131999969482422},{"id":"https://openalex.org/C21338462","wikidata":"https://www.wikidata.org/wiki/Q1662581","display_name":"Information model","level":2,"score":0.311599999666214},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.3068000078201294},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.30469998717308044},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2815999984741211},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.2736999988555908},{"id":"https://openalex.org/C20136886","wikidata":"https://www.wikidata.org/wiki/Q749647","display_name":"Interoperability","level":2,"score":0.265500009059906},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.26440000534057617},{"id":"https://openalex.org/C145644426","wikidata":"https://www.wikidata.org/wiki/Q169411","display_name":"Unified Modeling Language","level":3,"score":0.2531000077724457},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.25209999084472656}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3765611.3815141","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3765611.3815141","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Sustainability Week","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3765611.3815141","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3765611.3815141","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Sustainability Week","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W2984798626","https://openalex.org/W4388585532","https://openalex.org/W4390547604","https://openalex.org/W4391800306","https://openalex.org/W4391987669","https://openalex.org/W4392490408","https://openalex.org/W4393063881","https://openalex.org/W4394838745","https://openalex.org/W4396881559","https://openalex.org/W4399554941","https://openalex.org/W4402502904","https://openalex.org/W4403564265","https://openalex.org/W4403585381","https://openalex.org/W4403888240","https://openalex.org/W4405035082","https://openalex.org/W4406945373","https://openalex.org/W4407687097","https://openalex.org/W4408636306","https://openalex.org/W4412742928","https://openalex.org/W4414834784","https://openalex.org/W4415159865","https://openalex.org/W4415251277","https://openalex.org/W4415314082","https://openalex.org/W4416125801","https://openalex.org/W4416888628","https://openalex.org/W6910454782","https://openalex.org/W7102405885"],"related_works":[],"abstract_inverted_index":{"The":[0],"advancement":[1],"of":[2,34,54,78,132],"Large":[3],"Language":[4],"Models":[5],"(LLMs)":[6],"has":[7],"raised":[8],"concerns":[9],"regarding":[10],"their":[11,121],"dual-use":[12],"potential":[13],"in":[14],"cybersecurity.":[15],"Existing":[16],"evaluation":[17,171],"frameworks":[18],"overwhelmingly":[19],"focus":[20],"on":[21,124],"Information":[22],"Technology":[23,36],"(IT)":[24],"environments,":[25],"failing":[26],"to":[27,49,150],"capture":[28],"the":[29,51,133,142],"constraints,":[30],"and":[31,72,89,116,146,170],"specialized":[32,155],"protocols":[33],"Operational":[35],"(OT).":[37],"To":[38,94],"address":[39],"this":[40,166],"gap,":[41],"we":[42,99],"introduce":[43],"CritBench,":[44],"a":[45,76,101],"novel":[46],"framework":[47],"designed":[48],"evaluate":[50],"cybersecurity":[52],"capabilities":[53],"LLM":[55],"agents":[56,110,158],"within":[57],"IEC":[58,134],"61850":[59,135],"Digital":[60],"Substation":[61],"environments.":[62],"We":[63],"assess":[64],"five":[65],"state-of-the-art":[66],"models,":[67,74],"including":[68],"OpenAI\u2019s":[69],"GPT-5":[70],"suite":[71],"open-weight":[73],"across":[75],"corpus":[77],"81":[79],"domain-specific":[80,102,161],"tasks":[81],"spanning":[82],"static":[83,113],"configuration":[84],"analysis,":[85],"network":[86,118],"traffic":[87],"reconnaissance,":[88],"live":[90,152],"virtual":[91],"machine":[92],"interaction.":[93],"facilitate":[95],"industrial":[96],"protocol":[97],"interaction,":[98],"develop":[100],"tool":[103,162],"scaffold.":[104],"Our":[105],"empirical":[106],"results":[107],"show":[108],"that":[109],"reliably":[111],"execute":[112],"structured-file":[114],"analysis":[115],"single-tool":[117],"enumeration,":[119],"but":[120],"performance":[122],"degrades":[123],"dynamic":[125],"tasks.":[126],"Despite":[127],"demonstrating":[128],"explicit,":[129],"internalized":[130],"knowledge":[131],"standards":[136],"terminology,":[137],"current":[138],"models":[139],"struggle":[140],"with":[141,159],"persistent":[143],"sequential":[144],"reasoning":[145],"state":[147],"tracking":[148],"required":[149],"manipulate":[151],"systems":[153],"without":[154],"tools.":[156],"Equipping":[157],"our":[160],"scaffold":[163],"significantly":[164],"mitigates":[165],"operational":[167],"bottleneck.":[168],"Code":[169],"scripts":[172],"are":[173],"available":[174],"at:":[175],"https://github.com/GKeppler/CritBench":[176]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-06-20T00:00:00"}