{"id":"https://openalex.org/W4414769424","doi":"https://doi.org/10.48550/arxiv.2505.00843","title":"OET: Optimization-based prompt injection Evaluation Toolkit","display_name":"OET: Optimization-based prompt injection Evaluation Toolkit","publication_year":2025,"publication_date":"2025-05-01","ids":{"openalex":"https://openalex.org/W4414769424","doi":"https://doi.org/10.48550/arxiv.2505.00843"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2505.00843","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.00843","pdf_url":"https://arxiv.org/pdf/2505.00843","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2505.00843","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Pan, Jinsheng","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pan, Jinsheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109332669","display_name":"Xiaogeng Liu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xiaogeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5005843046","display_name":"Chaowei Xiao","orcid":"https://orcid.org/0000-0002-7043-4926","observed_orcids":["https://orcid.org/0000-0002-7043-4926"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Chaowei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/prompt-injection","display_name":"prompt injection","score":0.9789999723434448},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8960000276565552},{"id":"https://openalex.org/keywords/adversarial-robustness","display_name":"adversarial robustness","score":0.8809999823570251},{"id":"https://openalex.org/keywords/prompt-injection-defense","display_name":"prompt injection defense","score":0.5870000123977661},{"id":"https://openalex.org/keywords/assessment-toolkit","display_name":"evaluation toolkit","score":0.6129999756813049},{"id":"https://openalex.org/keywords/white-box-and-black-box-attacks","display_name":"white-box and black-box attacks","score":0.7990000247955322},{"id":"https://openalex.org/keywords/red-teaming","display_name":"red teaming","score":0.7760000228881836},{"id":"https://openalex.org/keywords/security-assessment","display_name":"security analysis","score":0.3140000104904175}],"concepts":[{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.9312000274658203},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.73580002784729},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6917999982833862},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.6674000024795532},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4108000099658966},{"id":"https://openalex.org/C157486923","wikidata":"https://www.wikidata.org/wiki/Q1376436","display_name":"String (physics)","level":2,"score":0.35109999775886536},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3495999872684479},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.32420000433921814},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3154999911785126},{"id":"https://openalex.org/C195518309","wikidata":"https://www.wikidata.org/wiki/Q13424265","display_name":"Security testing","level":5,"score":0.30250000953674316},{"id":"https://openalex.org/C110251889","wikidata":"https://www.wikidata.org/wiki/Q1569697","display_name":"Model checking","level":2,"score":0.3010999858379364},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C147494362","wikidata":"https://www.wikidata.org/wiki/Q2078905","display_name":"Troubleshooting","level":2,"score":0.26429998874664307}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2505.00843","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.00843","pdf_url":"https://arxiv.org/pdf/2505.00843","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.00843","doi":"https://doi.org/10.48550/arxiv.2505.00843","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.00843","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2505.00843","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.00843","pdf_url":"https://arxiv.org/pdf/2505.00843","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7165736730","https://openalex.org/W4416391310","https://openalex.org/W7151294679","https://openalex.org/W7160944312","https://openalex.org/W7166844655","https://openalex.org/W4414581173","https://openalex.org/W4415011827","https://openalex.org/W4415314618","https://openalex.org/W4403812581","https://openalex.org/W4415968382"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"remarkable":[6],"capabilities":[7],"in":[8],"natural":[9],"language":[10],"understanding":[11],"and":[12,39,79,103,126],"generation,":[13,99],"enabling":[14,135],"their":[15,22,53],"widespread":[16],"adoption":[17],"across":[18,81],"various":[19],"domains.":[20],"However,":[21],"susceptibility":[23],"to":[24,50,129],"prompt":[25,76],"injection":[26,77],"attacks":[27,78],"poses":[28],"significant":[29],"security":[30,156],"risks,":[31],"as":[32],"adversarial":[33,58,97,113,132],"inputs":[34],"can":[35],"manipulate":[36],"model":[37],"behavior":[38],"override":[40],"intended":[41],"instructions.":[42],"Despite":[43],"numerous":[44],"defense":[45,146],"strategies,":[46],"a":[47,92,108],"standardized":[48],"framework":[49,119],"rigorously":[51],"evaluate":[52],"effectiveness,":[54],"especially":[55],"under":[56],"adaptive":[57,86,117],"scenarios,":[59],"is":[60],"lacking.":[61],"To":[62],"address":[63],"this":[64],"gap,":[65],"we":[66],"introduce":[67],"OET,":[68],"an":[69,85],"optimization-based":[70],"evaluation":[71],"toolkit":[72,90],"that":[73,95],"systematically":[74],"benchmarks":[75],"defenses":[80],"diverse":[82],"datasets":[83],"using":[84],"testing":[87,118],"framework.":[88],"Our":[89],"features":[91],"modular":[93],"workflow":[94],"facilitates":[96],"string":[98],"dynamic":[100],"attack":[101],"execution,":[102],"comprehensive":[104],"result":[105],"analysis,":[106],"offering":[107],"unified":[109],"platform":[110],"for":[111],"assessing":[112],"robustness.":[114],"Crucially,":[115],"the":[116,142],"leverages":[120],"optimization":[121],"methods":[122],"with":[123,148],"both":[124],"white-box":[125],"black-box":[127],"access":[128],"generate":[130],"worst-case":[131],"examples,":[133],"thereby":[134],"strict":[136],"red-teaming":[137],"evaluations.":[138],"Extensive":[139],"experiments":[140],"underscore":[141],"limitations":[143],"of":[144],"current":[145],"mechanisms,":[147],"some":[149],"models":[150],"remaining":[151],"susceptible":[152],"even":[153],"after":[154],"implementing":[155],"enhancements.":[157]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-09-26T08:19:03.415552","created_date":"2025-10-10T00:00:00"}