{"id":"https://openalex.org/W7138474525","doi":"https://doi.org/10.48550/arxiv.2603.14707","title":"Visual Confused Deputy: Exploiting and Defending Perception Failures in Computer-Using Agents","display_name":"Visual Confused Deputy: Exploiting and Defending Perception Failures in Computer-Using Agents","publication_year":2026,"publication_date":"2026-03-16","ids":{"openalex":"https://openalex.org/W7138474525","doi":"https://doi.org/10.48550/arxiv.2603.14707"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.14707","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.14707","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.14707","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030019120","display_name":"X Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xunzhuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129725759","display_name":"Bowei He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Bowei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129736771","display_name":"Xue Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129718402","display_name":"Andy Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Andy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129714648","display_name":"Haichen Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Haichen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129717509","display_name":"Huamin Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Huamin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.2906000018119812,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.2906000018119812,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.21320000290870667,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.09969999641180038,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.7039999961853027},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.6678000092506409},{"id":"https://openalex.org/keywords/object","display_name":"Object (grammar)","score":0.6100999712944031},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.49880000948905945},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4668999910354614},{"id":"https://openalex.org/keywords/channel","display_name":"Channel (broadcasting)","score":0.3804999887943268}],"concepts":[{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.7039999961853027},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6866999864578247},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.6678000092506409},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.6100999712944031},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.49880000948905945},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4668999910354614},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.447299987077713},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.44600000977516174},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.3804999887943268},{"id":"https://openalex.org/C178253425","wikidata":"https://www.wikidata.org/wiki/Q162668","display_name":"Visual perception","level":3,"score":0.34610000252723694},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3077999949455261},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.29019999504089355},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.28780001401901245},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.27639999985694885},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.27630001306533813},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.26840001344680786},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.26429998874664307},{"id":"https://openalex.org/C2780103172","wikidata":"https://www.wikidata.org/wiki/Q1309721","display_name":"Visual Objects","level":3,"score":0.2531999945640564}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.14707","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.14707","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.14707","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.14707","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.7576198577880859,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Computer-using":[0],"agents":[1],"(CUAs)":[2],"act":[3],"directly":[4],"on":[5,39,71],"graphical":[6],"user":[7],"interfaces,":[8],"yet":[9],"their":[10],"perception":[11],"of":[12,217],"the":[13,35,40,56,116,122,134,140,144,194,219],"screen":[14,74],"is":[15,37,49,89,161,223],"often":[16],"unreliable.":[17],"Existing":[18],"work":[19],"largely":[20],"treats":[21],"these":[22,163],"failures":[23],"as":[24],"performance":[25],"limitations,":[26],"asking":[27],"whether":[28,34],"an":[29,65,68],"action":[30,69,145,212],"succeeds,":[31],"rather":[32],"than":[33],"agent":[36,66,108,192,220],"acting":[38],"correct":[41],"object":[42],"at":[43],"all.":[44],"We":[45,54],"argue":[46],"that":[47,119,162,205],"this":[48,112],"fundamentally":[50],"a":[51,60,72],"security":[52],"problem.":[53],"formalize":[55],"visual":[57,135,170],"confused":[58],"deputy:":[59],"failure":[61,168],"mode":[62],"in":[63],"which":[64],"authorizes":[67],"based":[70],"misperceived":[73],"state,":[75],"due":[76],"to":[77],"grounding":[78],"errors,":[79],"adversarial":[80],"screenshot":[81],"manipulation,":[82],"or":[83],"time-of-check-to-time-of-use":[84],"(TOCTOU)":[85],"races.":[86],"This":[87],"gap":[88],"practically":[90],"exploitable:":[91],"even":[92],"simple":[93],"screen-level":[94],"manipulations":[95],"can":[96],"redirect":[97],"routine":[98],"clicks":[99],"into":[100],"privileged":[101],"actions":[102],"while":[103,175],"remaining":[104],"indistinguishable":[105],"from":[106],"ordinary":[107],"mistakes.":[109],"To":[110],"mitigate":[111],"threat,":[113],"we":[114],"propose":[115],"first":[117],"guardrail":[118,196],"operates":[120],"outside":[121],"agent's":[123,141],"perceptual":[124],"loop.":[125],"Our":[126,202],"method,":[127],"dual-channel":[128],"contrastive":[129],"classification,":[130],"independently":[131],"evaluates":[132],"(1)":[133],"click":[136],"target":[137],"and":[138,150,191,225,231],"(2)":[139],"reasoning":[142,177],"about":[143],"against":[146],"deployment-specific":[147],"knowledge":[148],"bases,":[149],"blocks":[151],"execution":[152],"if":[153],"either":[154,199],"channel":[155,200],"indicates":[156],"risk.":[157],"The":[158],"key":[159],"insight":[160],"two":[164],"channels":[165],"capture":[166],"complementary":[167],"modes:":[169],"evidence":[171],"detects":[172],"target-level":[173],"mismatches,":[174],"textual":[176],"reveals":[178],"dangerous":[179],"intent":[180],"behind":[181],"visually":[182],"innocuous":[183],"controls.":[184],"Across":[185],"controlled":[186],"attacks,":[187],"real":[188],"GUI":[189],"screenshots,":[190],"traces,":[193],"combined":[195],"consistently":[197],"outperforms":[198],"alone.":[201],"results":[203],"suggest":[204],"CUA":[206],"safety":[207],"requires":[208],"not":[209],"only":[210],"better":[211],"generation,":[213],"but":[214],"independent":[215],"verification":[216],"what":[218],"believes":[221],"it":[222],"clicking":[224],"why.":[226],"Materials":[227],"are":[228],"provided\\footnote{Model,":[229],"benchmark,":[230],"code:":[232],"https://github.com/vllm-project/semantic-router}.":[233]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-03-18T00:00:00"}