{"id":"https://openalex.org/W4403363409","doi":"https://doi.org/10.48550/arxiv.2410.06703","title":"ST-WebAgentBench: A Benchmark for Evaluating Safety and Trustworthiness in Web Agents","display_name":"ST-WebAgentBench: A Benchmark for Evaluating Safety and Trustworthiness in Web Agents","publication_year":2024,"publication_date":"2024-10-09","ids":{"openalex":"https://openalex.org/W4403363409","doi":"https://doi.org/10.48550/arxiv.2410.06703"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.06703","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2410.06703","pdf_url":"https://arxiv.org/pdf/2410.06703","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.06703","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108490147","display_name":"I. A. Levy","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Levy, Ido","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107649410","display_name":"Ben Wiesel","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wiesel, Ben","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5105103094","display_name":"Sami Marreed","orcid":"https://orcid.org/0009-0001-8704-2929","observed_orcids":["https://orcid.org/0009-0001-8704-2929"]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Marreed, Sami","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021969735","display_name":"Alon Oved","orcid":"https://orcid.org/0000-0002-9328-1467","observed_orcids":["https://orcid.org/0000-0002-9328-1467"]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Oved, Alon","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061821122","display_name":"Avi Yaeli","orcid":"https://orcid.org/0000-0003-4698-1927","observed_orcids":["https://orcid.org/0000-0003-4698-1927"]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yaeli, Avi","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Mashkif, Nir","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mashkif, Nir","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5107555313","display_name":"Segev Shlomov","orcid":"https://orcid.org/0000-0003-1216-8284","observed_orcids":["https://orcid.org/0000-0003-1216-8284"]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shlomov, Segev","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":132,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.7515000104904175,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.7515000104904175,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10927","display_name":"Access Control and Trust","score":0.033799998462200165,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.01640000008046627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.4909999966621399},{"id":"https://openalex.org/keywords/safety-policy","display_name":"safety policy","score":0.6430000066757202},{"id":"https://openalex.org/keywords/user-consent","display_name":"user consent","score":0.6759999990463257},{"id":"https://openalex.org/keywords/robustness","display_name":"robustness","score":0.8360000252723694},{"id":"https://openalex.org/keywords/relative-risk","display_name":"relative risk","score":0.9570000171661377},{"id":"https://openalex.org/keywords/enterprise-adoption","display_name":"enterprise adoption","score":0.5950000286102295},{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.5289999842643738}],"concepts":[{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.790508508682251},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7713645100593567},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5149543881416321},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.44089236855506897},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3379333019256592},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.31369054317474365},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.14375418424606323},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.09376901388168335}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.06703","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2410.06703","pdf_url":"https://arxiv.org/pdf/2410.06703","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.06703","doi":"https://doi.org/10.48550/arxiv.2410.06703","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.06703","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.06703","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2410.06703","pdf_url":"https://arxiv.org/pdf/2410.06703","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403363409.pdf","grobid_xml":"https://content.openalex.org/works/W4403363409.grobid-xml"},"referenced_works_count":37,"referenced_works":["https://openalex.org/W2786616516","https://openalex.org/W3167070827","https://openalex.org/W4385682136","https://openalex.org/W4385774833","https://openalex.org/W4385774857","https://openalex.org/W4385965642","https://openalex.org/W4388926678","https://openalex.org/W4390833061","https://openalex.org/W4391047179","https://openalex.org/W4392781346","https://openalex.org/W4394972055","https://openalex.org/W4399404790","https://openalex.org/W4399695705","https://openalex.org/W4399837154","https://openalex.org/W4402346179","https://openalex.org/W4402402057","https://openalex.org/W4402427562","https://openalex.org/W4402667024","https://openalex.org/W4403622204","https://openalex.org/W4404260034","https://openalex.org/W4404782768","https://openalex.org/W4405095100","https://openalex.org/W4405263848","https://openalex.org/W4405562840","https://openalex.org/W4406800520","https://openalex.org/W4407184838","https://openalex.org/W4411120146","https://openalex.org/W4412877234","https://openalex.org/W4415271252","https://openalex.org/W4415427913","https://openalex.org/W4415798926","https://openalex.org/W4416097304","https://openalex.org/W4416375749","https://openalex.org/W4417314204","https://openalex.org/W7006598167","https://openalex.org/W7131050341","https://openalex.org/W7165728955"],"related_works":["https://openalex.org/W7124151464","https://openalex.org/W7152705456","https://openalex.org/W6929290301","https://openalex.org/W7204562187","https://openalex.org/W7091187108","https://openalex.org/W4414968681","https://openalex.org/W7164973461","https://openalex.org/W7147545502","https://openalex.org/W4415154343","https://openalex.org/W7134905593"],"abstract_inverted_index":{"Autonomous":[0],"web":[1,58,163],"agents":[2,34,126,164],"solve":[3],"complex":[4],"browsing":[5],"tasks,":[6],"yet":[7],"existing":[8],"benchmarks":[9],"measure":[10],"only":[11,105],"whether":[12,19],"an":[13,156],"agent":[14,59],"finishes":[15],"a":[16,26,50,151],"task,":[17],"ignoring":[18],"it":[20],"does":[21],"so":[22],"safely":[23],"or":[24],"in":[25],"way":[27],"enterprises":[28],"can":[29],"trust.":[30],"To":[31],"integrate":[32],"these":[33],"into":[35],"critical":[36,142],"workflows,":[37],"safety":[38,143],"and":[39,52,80,112,150],"trustworthiness":[40],"(ST)":[41],"are":[42],"prerequisite":[43],"conditions":[44],"for":[45,56],"adoption.":[46],"We":[47],"introduce":[48],"\\textbf{\\textsc{ST-WebAgentBench}},":[49],"configurable":[51],"easily":[53],"extensible":[54],"suite":[55],"evaluating":[57],"ST":[60,73,118],"across":[61,120],"realistic":[62],"enterprise":[63],"scenarios.":[64],"Each":[65],"of":[66,136],"its":[67],"222":[68],"tasks":[69],"is":[70,81,132],"paired":[71],"with":[72],"policies,":[74,111],"concise":[75],"rules":[76],"that":[77,107,128],"encode":[78],"constraints,":[79],"scored":[82],"along":[83],"six":[84],"orthogonal":[85],"dimensions":[86],"(e.g.,":[87],"user":[88],"consent,":[89],"robustness).":[90],"Beyond":[91],"raw":[92],"task":[93],"success,":[94],"we":[95],"propose":[96],"the":[97,113],"\\textit{Completion":[98],"Under":[99],"Policy}":[100],"(\\textit{CuP})":[101],"metric,":[102],"which":[103,116],"credits":[104],"completions":[106],"respect":[108],"all":[109],"applicable":[110],"\\textit{Risk":[114],"Ratio},":[115],"quantifies":[117],"breaches":[119],"dimensions.":[121],"Evaluating":[122],"three":[123],"open":[124],"state-of-the-art":[125],"reveals":[127],"their":[129,137],"average":[130],"CuP":[131],"less":[133],"than":[134],"two-thirds":[135],"nominal":[138],"completion":[139],"rate,":[140],"exposing":[141],"gaps.":[144],"By":[145],"releasing":[146],"code,":[147],"evaluation":[148],"templates,":[149],"policy-authoring":[152],"interface,":[153],"\\href{https://sites.google.com/view/st-webagentbench/home}{\\textsc{ST-WebAgentBench}}":[154],"provides":[155],"actionable":[157],"first":[158],"step":[159],"toward":[160],"deploying":[161],"trustworthy":[162],"at":[165],"scale.":[166]},"counts_by_year":[{"year":2026,"cited_by_count":88},{"year":2025,"cited_by_count":41},{"year":2024,"cited_by_count":3}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-10-13T00:00:00"}