{"id":"https://openalex.org/W4403585381","doi":"https://doi.org/10.48550/arxiv.2408.01605","title":"CYBERSECEVAL 3: Advancing the Evaluation of Cybersecurity Risks and Capabilities in Large Language Models","display_name":"CYBERSECEVAL 3: Advancing the Evaluation of Cybersecurity Risks and Capabilities in Large Language Models","publication_year":2024,"publication_date":"2024-08-02","ids":{"openalex":"https://openalex.org/W4403585381","doi":"https://doi.org/10.48550/arxiv.2408.01605"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2408.01605","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.01605","pdf_url":"https://arxiv.org/pdf/2408.01605","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2408.01605","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5004300345","display_name":"Shengye Wan","orcid":"https://orcid.org/0000-0001-6117-4109","observed_orcids":["https://orcid.org/0000-0001-6117-4109"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wan, Shengye","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093470850","display_name":"Cyrus Nikolaidis","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nikolaidis, Cyrus","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111944920","display_name":"Daniel S. Song","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101905586","display_name":"D\u00e1vid Moln\u00e1r","orcid":"https://orcid.org/0000-0002-2398-4984","observed_orcids":["https://orcid.org/0000-0002-2398-4984"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Molnar, David","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114341084","display_name":"James Crnkovich","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Crnkovich, James","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Grace, Jayson","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Grace, Jayson","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112988008","display_name":"Manish Bhatt","orcid":"https://orcid.org/0000-0002-3867-5180","observed_orcids":["https://orcid.org/0000-0002-3867-5180"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bhatt, Manish","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093470849","display_name":"Sahana Chennabasappa","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chennabasappa, Sahana","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044903923","display_name":"Spencer Whitman","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Whitman, Spencer","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029233360","display_name":"Stephanie Ding","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Stephanie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106276430","display_name":"Vlad Ionescu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ionescu, Vlad","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115028368","display_name":"Yue Li","orcid":"https://orcid.org/0009-0000-7206-2436","observed_orcids":["https://orcid.org/0009-0000-7206-2436"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5055990462","display_name":"Joshua Saxe","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Saxe, Joshua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":135,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.002899999963119626,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8029999732971191},{"id":"https://openalex.org/keywords/llm-security","display_name":"LLM security","score":0.6629999876022339},{"id":"https://openalex.org/keywords/cybersecurity-risk-assessment","display_name":"cybersecurity risk assessment","score":0.800000011920929},{"id":"https://openalex.org/keywords/security-benchmarks","display_name":"security benchmarks","score":0.8180000185966492},{"id":"https://openalex.org/keywords/social-engineering","display_name":"social engineering","score":0.6330000162124634},{"id":"https://openalex.org/keywords/offensive-cyber-operations","display_name":"offensive cyber operations","score":0.7870000004768372},{"id":"https://openalex.org/keywords/llama-3","display_name":"Llama 3","score":0.859000027179718},{"id":"https://openalex.org/keywords/third-party-risk","display_name":"third-party risk","score":0.699999988079071},{"id":"https://openalex.org/keywords/security-capabilities","display_name":"security capabilities","score":0.3319999873638153}],"concepts":[{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.5979334712028503},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5341740250587463},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.37842926383018494},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32509881258010864},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.22744804620742798}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2408.01605","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.01605","pdf_url":"https://arxiv.org/pdf/2408.01605","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2408.01605","doi":"https://doi.org/10.48550/arxiv.2408.01605","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2408.01605","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2408.01605","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.01605","pdf_url":"https://arxiv.org/pdf/2408.01605","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403585381.pdf","grobid_xml":"https://content.openalex.org/works/W4403585381.grobid-xml"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W4376312605","https://openalex.org/W4389523893","https://openalex.org/W4389599215","https://openalex.org/W4389713766","https://openalex.org/W4392848643","https://openalex.org/W4394838745","https://openalex.org/W4395064704","https://openalex.org/W4402670748","https://openalex.org/W4408941302"],"related_works":["https://openalex.org/W4395064704","https://openalex.org/W7154606467","https://openalex.org/W4402546694","https://openalex.org/W7211912521","https://openalex.org/W4408565663","https://openalex.org/W7162027986","https://openalex.org/W4412608681","https://openalex.org/W4389599215","https://openalex.org/W4407122991","https://openalex.org/W4403564265"],"abstract_inverted_index":{"We":[0],"are":[1],"releasing":[2],"a":[3,87],"new":[4,53],"suite":[5,88],"of":[6,89],"security":[7,58],"benchmarks":[8,80],"for":[9],"LLMs,":[10,92],"CYBERSECEVAL":[11,25],"3,":[12],"to":[13,36,41,48,81,95],"continue":[14],"the":[15,82],"conversation":[16],"on":[17,56],"empirically":[18],"measuring":[19],"LLM":[20],"cybersecurity":[21],"risks":[22,30,97],"and":[23,39,44,68,86,100],"capabilities.":[24],"3":[26,84],"assesses":[27],"8":[28],"different":[29],"across":[31],"two":[32],"broad":[33],"categories:":[34],"risk":[35,40],"third":[37],"parties,":[38],"application":[42],"developers":[43],"end":[45],"users.":[46],"Compared":[47],"previous":[49],"work,":[50],"we":[51,76],"add":[52],"areas":[54],"focused":[55],"offensive":[57,65,70],"capabilities:":[59],"automated":[60],"social":[61],"engineering,":[62],"scaling":[63],"manual":[64],"cyber":[66,71],"operations,":[67],"autonomous":[69],"operations.":[72],"In":[73],"this":[74],"paper":[75],"discuss":[77],"applying":[78],"these":[79],"Llama":[83],"models":[85],"contemporaneous":[90],"state-of-the-art":[91],"enabling":[93],"us":[94],"contextualize":[96],"both":[98],"with":[99],"without":[101],"mitigations":[102],"in":[103],"place.":[104]},"counts_by_year":[{"year":2026,"cited_by_count":76},{"year":2025,"cited_by_count":53},{"year":2024,"cited_by_count":6}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2024-10-21T00:00:00"}