{"id":"https://openalex.org/W7159630839","doi":"https://doi.org/10.48550/arxiv.2604.27601","title":"SecGoal: A Benchmark for Extracting Formalizable Security Goals from Protocol Documents","display_name":"SecGoal: A Benchmark for Extracting Formalizable Security Goals from Protocol Documents","publication_year":2026,"publication_date":"2026-04-30","ids":{"openalex":"https://openalex.org/W7159630839","doi":"https://doi.org/10.48550/arxiv.2604.27601"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.27601","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.27601","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.27601","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134987133","display_name":"Dawei Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Dawei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065859286","display_name":"Hui Li","orcid":"https://orcid.org/0000-0001-9198-3951"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Hui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134956609","display_name":"Bo Jia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jia, Bo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056537907","display_name":"Haonan Feng","orcid":"https://orcid.org/0009-0002-0140-8342"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feng, Haonan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Guan, Jingjing","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guan, Jingjing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134956609","display_name":"Bo Jia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiao, Yueshuang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Li, Xiangdong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xiangdong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11504","display_name":"Advanced Authentication Protocols Security","score":0.4456000030040741,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11504","display_name":"Advanced Authentication Protocols Security","score":0.4456000030040741,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10237","display_name":"Cryptography and Data Security","score":0.07819999754428864,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11800","display_name":"User Authentication and Security Systems","score":0.0674000009894371,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.6765999794006348},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6176999807357788},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5516999959945679},{"id":"https://openalex.org/keywords/cryptographic-protocol","display_name":"Cryptographic protocol","score":0.511900007724762},{"id":"https://openalex.org/keywords/bridging","display_name":"Bridging (networking)","score":0.5070000290870667},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.4007999897003174},{"id":"https://openalex.org/keywords/cryptography","display_name":"Cryptography","score":0.3889000117778778},{"id":"https://openalex.org/keywords/information-extraction","display_name":"Information extraction","score":0.3472000062465668}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8159999847412109},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.6765999794006348},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6176999807357788},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5516999959945679},{"id":"https://openalex.org/C33884865","wikidata":"https://www.wikidata.org/wiki/Q1254335","display_name":"Cryptographic protocol","level":3,"score":0.511900007724762},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.5070000290870667},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.4007999897003174},{"id":"https://openalex.org/C178489894","wikidata":"https://www.wikidata.org/wiki/Q8789","display_name":"Cryptography","level":2,"score":0.3889000117778778},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3546000123023987},{"id":"https://openalex.org/C195807954","wikidata":"https://www.wikidata.org/wiki/Q1662562","display_name":"Information extraction","level":2,"score":0.3472000062465668},{"id":"https://openalex.org/C75606506","wikidata":"https://www.wikidata.org/wiki/Q1049183","display_name":"Formal methods","level":2,"score":0.34450000524520874},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3411000072956085},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3407999873161316},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.3382999897003174},{"id":"https://openalex.org/C2985583900","wikidata":"https://www.wikidata.org/wiki/Q722617","display_name":"Formal description","level":2,"score":0.33000001311302185},{"id":"https://openalex.org/C111498074","wikidata":"https://www.wikidata.org/wiki/Q173326","display_name":"Formal verification","level":2,"score":0.32109999656677246},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.319599986076355},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.31360000371932983},{"id":"https://openalex.org/C116253237","wikidata":"https://www.wikidata.org/wiki/Q1437424","display_name":"Formal specification","level":2,"score":0.30809998512268066},{"id":"https://openalex.org/C146072743","wikidata":"https://www.wikidata.org/wiki/Q192161","display_name":"Formal language","level":2,"score":0.2971999943256378},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.28029999136924744},{"id":"https://openalex.org/C2779639559","wikidata":"https://www.wikidata.org/wiki/Q7661178","display_name":"Symbolic execution","level":3,"score":0.26100000739097595},{"id":"https://openalex.org/C195344581","wikidata":"https://www.wikidata.org/wiki/Q2555318","display_name":"Automated reasoning","level":2,"score":0.25870001316070557},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.25290000438690186},{"id":"https://openalex.org/C109747225","wikidata":"https://www.wikidata.org/wiki/Q815758","display_name":"Scarcity","level":2,"score":0.25099998712539673},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.25060001015663147},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.25049999356269836}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.27601","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.27601","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.27601","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.27601","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Formal":[0],"verification":[1],"provides":[2],"rigorous":[3],"guarantees":[4],"for":[5,29,51,164],"cryptographic":[6],"security,":[7],"yet":[8],"extracting":[9,30],"formalizable":[10,31,79,99],"security":[11,32,54,80,100,166],"goals":[12,81],"from":[13,35,82,111],"natural-language":[14],"protocol":[15,36,84],"documents":[16],"remains":[17],"largely":[18],"manual.":[19],"We":[20],"introduce":[21],"SecGoal,":[22],"a":[23,46,128,159],"dedicated":[24],"expert-annotated":[25],"dataset":[26],"and":[27,48,62,115,124,156,162,169],"benchmark":[28],"goal":[33,135,167],"statements":[34],"documents,":[37],"covering":[38],"15":[39],"widely":[40],"deployed":[41],"protocols,":[42,106],"together":[43],"with":[44],"AIFG,":[45],"schema-":[47],"flow-conditioned":[49],"framework":[50,163],"structured":[52,140],"formal":[53],"property":[55,67,118,141,170],"generation.":[56,171],"Our":[57],"evaluation":[58],"shows":[59,132],"that":[60,133],"frontier":[61],"large":[63],"LLMs":[64,93,123],"achieve":[65],"high":[66],"recall":[68],"but":[69],"low":[70],"extraction":[71,109,168],"precision":[72,110],"because":[73],"they":[74],"often":[75],"fail":[76],"to":[77,113],"distinguish":[78],"non-goal":[83],"content.":[85],"In":[86,127],"contrast,":[87],"SecGoal":[88,155],"fine-tuning":[89],"makes":[90],"smaller":[91],"open-source":[92],"substantially":[94],"more":[95],"selective":[96],"extractors":[97],"of":[98],"goals.":[101],"On":[102],"the":[103,150],"held-out":[104],"test":[105],"Gemma2-9B-FT":[107],"improves":[108],"24.0\\%":[112],"66.6\\%":[114],"reaches":[116],"97.6\\%":[117],"recall,":[119],"outperforming":[120],"larger":[121],"prompted":[122],"encoder":[125],"baselines.":[126],"controlled":[129],"setting,":[130],"AIFG":[131,157],"concise":[134],"inputs":[136,146],"can":[137],"support":[138],"high-recall":[139],"generation,":[142],"while":[143],"expert-vetted":[144],"extracted":[145],"reveal":[147],"over-generation":[148],"as":[149],"main":[151],"remaining":[152],"bottleneck.":[153],"Together,":[154],"provide":[158],"dataset,":[160],"benchmark,":[161],"specification-grounded":[165]},"counts_by_year":[],"updated_date":"2026-07-21T08:15:58.654021","created_date":"2026-05-02T00:00:00"}