{"id":"https://openalex.org/W7151779087","doi":"https://doi.org/10.48550/arxiv.2604.06019","title":"CritBench: A Framework for Evaluating Cybersecurity Capabilities of Large Language Models in IEC 61850 Digital Substation Environments","display_name":"CritBench: A Framework for Evaluating Cybersecurity Capabilities of Large Language Models in IEC 61850 Digital Substation Environments","publication_year":2026,"publication_date":"2026-04-07","ids":{"openalex":"https://openalex.org/W7151779087","doi":"https://doi.org/10.48550/arxiv.2604.06019"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.06019","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.06019","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.06019","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100022307","display_name":"Gustav Keppler","orcid":"https://orcid.org/0000-0002-2323-0533"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Keppler, Gustav","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098085533","display_name":"Moritz Gst\u00fcr","orcid":"https://orcid.org/0009-0002-4036-9032"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gst\u00fcr, Moritz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133233197","display_name":"Veit Hagenmeyer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hagenmeyer, Veit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10917","display_name":"Smart Grid Security and Resilience","score":0.30079999566078186,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10917","display_name":"Smart Grid Security and Resilience","score":0.30079999566078186,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.20319999754428864,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.05719999969005585,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/iec-61850","display_name":"IEC 61850","score":0.8575999736785889},{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.6370999813079834},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4943000078201294},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.454800009727478},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.45019999146461487},{"id":"https://openalex.org/keywords/interoperability","display_name":"Interoperability","score":0.4390000104904175},{"id":"https://openalex.org/keywords/communications-protocol","display_name":"Communications protocol","score":0.37310001254081726},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.37040001153945923}],"concepts":[{"id":"https://openalex.org/C2778907243","wikidata":"https://www.wikidata.org/wiki/Q168160","display_name":"IEC 61850","level":3,"score":0.8575999736785889},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6417999863624573},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.6370999813079834},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4943000078201294},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.47929999232292175},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.454800009727478},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.45019999146461487},{"id":"https://openalex.org/C20136886","wikidata":"https://www.wikidata.org/wiki/Q749647","display_name":"Interoperability","level":2,"score":0.4390000104904175},{"id":"https://openalex.org/C12269588","wikidata":"https://www.wikidata.org/wiki/Q132364","display_name":"Communications protocol","level":2,"score":0.37310001254081726},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.37040001153945923},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.3695000112056732},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.3659999966621399},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.3386000096797943},{"id":"https://openalex.org/C2781045450","wikidata":"https://www.wikidata.org/wiki/Q254569","display_name":"Backdoor","level":2,"score":0.3287999927997589},{"id":"https://openalex.org/C25344961","wikidata":"https://www.wikidata.org/wiki/Q192726","display_name":"Virtual machine","level":2,"score":0.3109000027179718},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.3107999861240387},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.30379998683929443},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.2904999852180481},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2741999924182892},{"id":"https://openalex.org/C21338462","wikidata":"https://www.wikidata.org/wiki/Q1662581","display_name":"Information model","level":2,"score":0.26919999718666077},{"id":"https://openalex.org/C145804949","wikidata":"https://www.wikidata.org/wiki/Q478123","display_name":"Situation awareness","level":2,"score":0.2558000087738037},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.25200000405311584}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.06019","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.06019","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.06019","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.06019","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"advancement":[1],"of":[2,34,54,78,132],"Large":[3],"Language":[4],"Models":[5],"(LLMs)":[6],"has":[7],"raised":[8],"concerns":[9],"regarding":[10],"their":[11,121],"dual-use":[12],"potential":[13],"in":[14],"cybersecurity.":[15],"Existing":[16],"evaluation":[17,171],"frameworks":[18],"overwhelmingly":[19],"focus":[20],"on":[21,124],"Information":[22],"Technology":[23,36],"(IT)":[24],"environments,":[25],"failing":[26],"to":[27,49,150],"capture":[28],"the":[29,51,133,142],"constraints,":[30],"and":[31,72,89,116,146,170],"specialized":[32,155],"protocols":[33],"Operational":[35],"(OT).":[37],"To":[38,94],"address":[39],"this":[40,166],"gap,":[41],"we":[42,99],"introduce":[43],"CritBench,":[44],"a":[45,76,101],"novel":[46],"framework":[47],"designed":[48],"evaluate":[50],"cybersecurity":[52],"capabilities":[53],"LLM":[55],"agents":[56,110,158],"within":[57],"IEC":[58,134],"61850":[59,135],"Digital":[60],"Substation":[61],"environments.":[62],"We":[63],"assess":[64],"five":[65],"state-of-the-art":[66],"models,":[67,74],"including":[68],"OpenAI's":[69],"GPT-5":[70],"suite":[71],"open-weight":[73],"across":[75],"corpus":[77],"81":[79],"domain-specific":[80,102,161],"tasks":[81],"spanning":[82],"static":[83,113],"configuration":[84],"analysis,":[85],"network":[86,118],"traffic":[87],"reconnaissance,":[88],"live":[90,152],"virtual":[91],"machine":[92],"interaction.":[93],"facilitate":[95],"industrial":[96],"protocol":[97],"interaction,":[98],"develop":[100],"tool":[103,162],"scaffold.":[104],"Our":[105],"empirical":[106],"results":[107],"show":[108],"that":[109],"reliably":[111],"execute":[112],"structured-file":[114],"analysis":[115],"single-tool":[117],"enumeration,":[119],"but":[120],"performance":[122],"degrades":[123],"dynamic":[125],"tasks.":[126],"Despite":[127],"demonstrating":[128],"explicit,":[129],"internalized":[130],"knowledge":[131],"standards":[136],"terminology,":[137],"current":[138],"models":[139],"struggle":[140],"with":[141,159],"persistent":[143],"sequential":[144],"reasoning":[145],"state":[147],"tracking":[148],"required":[149],"manipulate":[151],"systems":[153],"without":[154],"tools.":[156],"Equipping":[157],"our":[160],"scaffold":[163],"significantly":[164],"mitigates":[165],"operational":[167],"bottleneck.":[168],"Code":[169],"scripts":[172],"are":[173],"available":[174],"at:":[175],"https://github.com/GKeppler/CritBench":[176]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-09T00:00:00"}