{"id":"https://openalex.org/W7164131171","doi":"https://doi.org/10.48550/arxiv.2606.10484","title":"AgentCanary: A Security Evaluation Framework for Autonomous AI Agents in Real Executable Environments","display_name":"AgentCanary: A Security Evaluation Framework for Autonomous AI Agents in Real Executable Environments","publication_year":2026,"publication_date":"2026-06-09","ids":{"openalex":"https://openalex.org/W7164131171","doi":"https://doi.org/10.48550/arxiv.2606.10484"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.10484","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.10484","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.10484","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138330034","display_name":"Peiyang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Peiyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091587922","display_name":"Songping Wang","orcid":"https://orcid.org/0009-0001-4513-7284"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Songping","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138368726","display_name":"Yi C. Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Yi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138391534","display_name":"Yanhua Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Yanhua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138334723","display_name":"Chenhao Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Chenhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138354873","display_name":"Qi Li (67548)","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Qi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138348876","display_name":"Yueming Lyu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lyu, Yueming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138365469","display_name":"Caifeng Shan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shan, Caifeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101774044","display_name":"Fengting Li","orcid":"https://orcid.org/0000-0002-4839-3435"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Fengting","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138336364","display_name":"Chao Feng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feng, Chao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138326872","display_name":"Chuanqun Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Chuanqun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138339025","display_name":"Liang Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Liang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.5543000102043152,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.5543000102043152,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.06960000097751617,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.06809999793767929,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5414000153541565},{"id":"https://openalex.org/keywords/executable","display_name":"Executable","score":0.510699987411499},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.4779999852180481},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4097000062465668},{"id":"https://openalex.org/keywords/provisioning","display_name":"Provisioning","score":0.3750999867916107},{"id":"https://openalex.org/keywords/computer-security-model","display_name":"Computer security model","score":0.3398999869823456},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.33009999990463257},{"id":"https://openalex.org/keywords/intelligent-agent","display_name":"Intelligent agent","score":0.325300008058548}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7699999809265137},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5414000153541565},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.510699987411499},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.47839999198913574},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.4779999852180481},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4097000062465668},{"id":"https://openalex.org/C172191483","wikidata":"https://www.wikidata.org/wiki/Q1071806","display_name":"Provisioning","level":2,"score":0.3750999867916107},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3409999907016754},{"id":"https://openalex.org/C121822524","wikidata":"https://www.wikidata.org/wiki/Q5157582","display_name":"Computer security model","level":2,"score":0.3398999869823456},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.33009999990463257},{"id":"https://openalex.org/C74072328","wikidata":"https://www.wikidata.org/wiki/Q1142726","display_name":"Intelligent agent","level":2,"score":0.325300008058548},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.3197999894618988},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.31779998540878296},{"id":"https://openalex.org/C13687954","wikidata":"https://www.wikidata.org/wiki/Q4826847","display_name":"Autonomous agent","level":2,"score":0.31279999017715454},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.31049999594688416},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.3102000057697296},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2994999885559082},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.2858999967575073},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.2849000096321106},{"id":"https://openalex.org/C2780264999","wikidata":"https://www.wikidata.org/wiki/Q7445032","display_name":"Security domain","level":2,"score":0.2831999957561493},{"id":"https://openalex.org/C41550386","wikidata":"https://www.wikidata.org/wiki/Q529909","display_name":"Multi-agent system","level":2,"score":0.28049999475479126},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.2711000144481659},{"id":"https://openalex.org/C51929080","wikidata":"https://www.wikidata.org/wiki/Q2425187","display_name":"Codebase","level":3,"score":0.2574999928474426}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.10484","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.10484","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.10484","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.10484","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Autonomous":[0],"AI":[1,66],"agents":[2,129,210],"have":[3],"driven":[4],"the":[5,95,157,163,215],"transition":[6],"from":[7,15,97],"conversation":[8],"to":[9,18,213],"task":[10,109,137],"execution,":[11],"shifting":[12],"security":[13,22,61],"failures":[14],"textual":[16],"deception":[17],"system":[19],"compromise.":[20],"Although":[21],"evaluation":[23,62,155],"is":[24,32],"crucial":[25],"for":[26,64,234],"proactive":[27],"risk":[28,39,78,87],"prevention,":[29],"prior":[30],"work":[31],"constrained":[33],"by":[34],"fundamental":[35],"bottlenecks,":[36],"including":[37],"fragmented":[38],"coverage,":[40],"static":[41,123],"or":[42,125,166],"low-fidelity":[43],"execution":[44,227],"environments,":[45],"and":[46,48,103,182,225,229,238],"single-dimensional":[47],"coarse-grained":[49],"assessment":[50],"metrics.":[51],"To":[52],"address":[53],"these":[54],"challenges,":[55],"we":[56,80],"propose":[57],"AgentCanary,":[58],"a":[59,70,107,116,167,187,231],"comprehensive":[60,77],"framework":[63],"autonomous":[65],"agents.":[67],"AgentCanary":[68,194],"provides":[69],"systematic":[71,232],"solution":[72],"along":[73,174],"three":[74,175,202],"contributions.":[75],"First,":[76],"coverage:":[79],"introduce":[81],"an":[82],"orthogonal":[83,176],"Entry":[84],"$\\times$":[85],"Impact":[86],"taxonomy":[88],"that":[89,145,208],"decouples":[90],"how":[91],"adversarial":[92,198],"influence":[93],"enters":[94],"agent":[96,159,203,240],"what":[98],"harm":[99],"it":[100,105],"ultimately":[101],"causes,":[102],"instantiate":[104],"as":[106],"scenario-aligned":[108],"suite":[110],"spanning":[111],"realistic":[112],"deployment":[113],"workflows.":[114],"Second,":[115],"high-fidelity":[117],"real":[118,132],"executable":[119],"environment:":[120],"rather":[121,161],"than":[122,162],"Q&A":[124],"mocked":[126],"tool":[127,169],"responses,":[128],"interact":[130],"with":[131,139],"tools":[133],"against":[134,195],"dynamically":[135],"provisioned":[136],"artifacts,":[138],"persistent":[140,223],"state":[141],"across":[142,201],"multi-step":[143],"interactions":[144],"naturally":[146],"supports":[147],"long-horizon":[148,226],"attack":[149,199],"evaluation.":[150],"Third,":[151],"trajectory-grounded":[152],"multi-dimensional":[153],"evaluation:":[154],"consumes":[156],"full":[158],"trajectory":[160],"reply":[164],"text":[165],"single":[168],"call,":[170],"enabling":[171],"decomposed":[172],"scoring":[173],"dimensions,":[177],"Outcome":[178],"Safety,":[179],"Security":[180],"Awareness,":[181],"Task":[183],"Utility.":[184],"We":[185],"evaluate":[186],"broad":[188],"set":[189],"of":[190],"frontier":[191],"models":[192],"on":[193],"multiple":[196],"established":[197],"methods":[200],"frameworks.":[204],"The":[205],"results":[206],"reveal":[207],"current":[209],"often":[211],"fail":[212],"recognize":[214],"attacks":[216],"they":[217],"face,":[218],"particularly":[219],"under":[220],"compromised":[221],"skills,":[222],"state,":[224],"attacks,":[228],"provide":[230],"baseline":[233],"developing":[235],"more":[236],"reliable":[237],"secure":[239],"systems.":[241]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-11T00:00:00"}