{"id":"https://openalex.org/W4411523046","doi":"https://doi.org/10.1145/3728879","title":"Are Autonomous Web Agents Good Testers?","display_name":"Are Autonomous Web Agents Good Testers?","publication_year":2025,"publication_date":"2025-06-22","ids":{"openalex":"https://openalex.org/W4411523046","doi":"https://doi.org/10.1145/3728879"},"language":"en","primary_location":{"id":"doi:10.1145/3728879","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3728879","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3728879","source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3728879","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042093754","display_name":"Antoine Chevrot","orcid":"https://orcid.org/0000-0003-3677-5150"},"institutions":[{"id":"https://openalex.org/I4210107593","display_name":"Smartesting (France)","ror":"https://ror.org/01x1nsf94","country_code":"FR","type":"company","lineage":["https://openalex.org/I4210107593"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Antoine Chevrot","raw_affiliation_strings":["Smartesting, Besan\u00e7on, France"],"raw_orcid":"https://orcid.org/0000-0003-3677-5150","affiliations":[{"raw_affiliation_string":"Smartesting, Besan\u00e7on, France","institution_ids":["https://openalex.org/I4210107593"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082150617","display_name":"Alexandre Vernotte","orcid":"https://orcid.org/0000-0002-2113-4900"},"institutions":[{"id":"https://openalex.org/I4210107593","display_name":"Smartesting (France)","ror":"https://ror.org/01x1nsf94","country_code":"FR","type":"company","lineage":["https://openalex.org/I4210107593"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Alexandre Vernotte","raw_affiliation_strings":["Smartesting, Besan\u00e7on, France"],"raw_orcid":"https://orcid.org/0000-0002-2113-4900","affiliations":[{"raw_affiliation_string":"Smartesting, Besan\u00e7on, France","institution_ids":["https://openalex.org/I4210107593"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060648416","display_name":"Jean\u2010R\u00e9my Falleri","orcid":"https://orcid.org/0000-0002-8284-7218"},"institutions":[{"id":"https://openalex.org/I15057530","display_name":"Universit\u00e9 de Bordeaux","ror":"https://ror.org/057qpr032","country_code":"FR","type":"education","lineage":["https://openalex.org/I15057530"]},{"id":"https://openalex.org/I4210142254","display_name":"Laboratoire Bordelais de Recherche en Informatique","ror":"https://ror.org/03adqg323","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I15057530","https://openalex.org/I4210142254","https://openalex.org/I4210159245","https://openalex.org/I4210160189"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Jean-R\u00e9my Falleri","raw_affiliation_strings":["University of Bordeaux - LaBRI - UMR 5800, Bordeaux, France"],"raw_orcid":"https://orcid.org/0000-0002-8284-7218","affiliations":[{"raw_affiliation_string":"University of Bordeaux - LaBRI - UMR 5800, Bordeaux, France","institution_ids":["https://openalex.org/I15057530","https://openalex.org/I4210142254"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5105230109","display_name":"Xavier Blanc","orcid":"https://orcid.org/0000-0003-1783-0708"},"institutions":[{"id":"https://openalex.org/I15057530","display_name":"Universit\u00e9 de Bordeaux","ror":"https://ror.org/057qpr032","country_code":"FR","type":"education","lineage":["https://openalex.org/I15057530"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Xavier Blanc","raw_affiliation_strings":["University of Bordeaux, Bordeaux, France"],"raw_orcid":"https://orcid.org/0000-0003-1783-0708","affiliations":[{"raw_affiliation_string":"University of Bordeaux, Bordeaux, France","institution_ids":["https://openalex.org/I15057530"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082306957","display_name":"Bruno Legeard","orcid":"https://orcid.org/0000-0003-4986-7097"},"institutions":[{"id":"https://openalex.org/I4210107593","display_name":"Smartesting (France)","ror":"https://ror.org/01x1nsf94","country_code":"FR","type":"company","lineage":["https://openalex.org/I4210107593"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Bruno Legeard","raw_affiliation_strings":["Smartesting, Besan\u00e7on, France"],"raw_orcid":"https://orcid.org/0000-0003-4986-7097","affiliations":[{"raw_affiliation_string":"Smartesting, Besan\u00e7on, France","institution_ids":["https://openalex.org/I4210107593"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032190977","display_name":"Aymeric Cretin","orcid":null},"institutions":[{"id":"https://openalex.org/I4210107593","display_name":"Smartesting (France)","ror":"https://ror.org/01x1nsf94","country_code":"FR","type":"company","lineage":["https://openalex.org/I4210107593"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Aymeric Cretin","raw_affiliation_strings":["Smartesting, Besan\u00e7on, France"],"raw_orcid":"https://orcid.org/0009-0006-7686-136X","affiliations":[{"raw_affiliation_string":"Smartesting, Besan\u00e7on, France","institution_ids":["https://openalex.org/I4210107593"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.35,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.96942766,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":"2","issue":"ISSTA","first_page":"206","last_page":"228"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8134198188781738},{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.6484858989715576},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.5408191680908203},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.5059992671012878},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.5014047622680664},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.4959965646266937},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.49309250712394714},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.4803227484226227},{"id":"https://openalex.org/keywords/web-application","display_name":"Web application","score":0.471231073141098},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.4617045521736145},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3305419087409973},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3196153938770294},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.30976423621177673},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.1856781244277954}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8134198188781738},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.6484858989715576},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5408191680908203},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.5059992671012878},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.5014047622680664},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.4959965646266937},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.49309250712394714},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4803227484226227},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.471231073141098},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.4617045521736145},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3305419087409973},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3196153938770294},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.30976423621177673},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.1856781244277954},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3728879","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3728879","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3728879","source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3728879","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3728879","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3728879","source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320316512","display_name":"Bpifrance","ror":"https://ror.org/008zkt807"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4411523046.pdf","grobid_xml":"https://content.openalex.org/works/W4411523046.grobid-xml"},"referenced_works_count":44,"referenced_works":["https://openalex.org/W114331032","https://openalex.org/W1552573782","https://openalex.org/W1966956837","https://openalex.org/W2056112725","https://openalex.org/W2078924106","https://openalex.org/W2102218101","https://openalex.org/W2105714739","https://openalex.org/W2416751804","https://openalex.org/W2508918698","https://openalex.org/W2546669591","https://openalex.org/W2898893292","https://openalex.org/W2906640661","https://openalex.org/W2961883907","https://openalex.org/W3105878189","https://openalex.org/W3128169615","https://openalex.org/W3163125120","https://openalex.org/W3172076064","https://openalex.org/W4221143046","https://openalex.org/W4229772528","https://openalex.org/W4378373665","https://openalex.org/W4380186500","https://openalex.org/W4380374951","https://openalex.org/W4385327683","https://openalex.org/W4385570040","https://openalex.org/W4387800098","https://openalex.org/W4388247874","https://openalex.org/W4390188013","https://openalex.org/W4390306873","https://openalex.org/W4391272423","https://openalex.org/W4391974543","https://openalex.org/W4393065402","https://openalex.org/W4394710846","https://openalex.org/W4394769323","https://openalex.org/W4399668118","https://openalex.org/W4400396852","https://openalex.org/W4401856679","https://openalex.org/W4402400750","https://openalex.org/W4402667126","https://openalex.org/W4402706159","https://openalex.org/W4403536268","https://openalex.org/W4403578677","https://openalex.org/W4405627213","https://openalex.org/W4406755521","https://openalex.org/W4412203333"],"related_works":["https://openalex.org/W2170501285","https://openalex.org/W1579239352","https://openalex.org/W1598160211","https://openalex.org/W4253521920","https://openalex.org/W2179621094","https://openalex.org/W2067499928","https://openalex.org/W1524028411","https://openalex.org/W2138059712","https://openalex.org/W2018145554","https://openalex.org/W2127248783"],"abstract_inverted_index":{"Despite":[0],"advances":[1],"in":[2,25,30],"automated":[3,66],"testing,":[4],"manual":[5,115],"testing":[6],"remains":[7],"prevalent":[8],"due":[9],"to":[10,74,96,124,164,168,191,215,226,229],"the":[11,62,83,170,237],"high":[12],"maintenance":[13,242],"demands":[14],"associated":[15],"with":[16,22,49,101],"test":[17,91,116,141,199,243],"script":[18],"fragility\u2014scripts":[19],"often":[20],"break":[21],"minor":[23],"changes":[24],"application":[26],"structure.":[27],"Recent":[28],"developments":[29],"Large":[31],"Language":[32],"Models":[33],"(LLMs)":[34],"offer":[35],"a":[36,103,111,165,216],"potential":[37],"alternative":[38],"by":[39,68,77],"powering":[40],"Autonomous":[41,56],"Web":[42],"Agents":[43,58],"(AWAs)":[44],"that":[45,155,181,224],"can":[46],"autonomously":[47],"interact":[48],"applications.":[50],"These":[51],"agents":[52],"may":[53],"serve":[54],"as":[55],"Test":[57],"(ATAs),":[59],"potentially":[60],"reducing":[61],"need":[63,225],"for":[64,88,239],"maintenance-heavy":[65],"scripts":[67],"utilising":[69],"natural":[70,89],"language":[71,90],"instructions":[72],"similar":[73],"those":[75],"used":[76],"human":[78],"testers.":[79],"This":[80],"paper":[81],"investigates":[82],"feasibility":[84],"of":[85,105,113,139,172,210],"adapting":[86],"AWAs":[87],"case":[92],"execution":[93],"and":[94,110,121,126,132,145,148,213,233],"how":[95],"evaluate":[97,125],"them.":[98],"We":[99],"contribute":[100],"(1)":[102],"benchmark":[104,154],"three":[106],"offline":[107],"web":[108],"applications,":[109],"suite":[112],"113":[114],"cases,":[117,123],"split":[118],"between":[119],"passing":[120],"failing":[122],"compare":[127],"ATAs":[128,158],"performance,":[129],"(2)":[130],"SeeAct-ATA":[131],"pinATA,":[133],"two":[134],"open-source":[135],"ATA":[136],"implementations":[137],"capable":[138],"executing":[140,198],"steps,":[142],"verifying":[143],"assertions":[144],"giving":[146],"verdicts,":[147],"(3)":[149],"comparative":[150],"experiments":[151],"using":[152],"our":[153,157,174,182,192],"quantifies":[156],"effectiveness.":[159],"Finally":[160],"we":[161,220],"also":[162],"proceed":[163],"qualitative":[166],"evaluation":[167],"identify":[169,221],"limitations":[171,223],"PinATA,":[173],"best":[175],"performing":[176],"implementation.":[177],"Our":[178],"findings":[179],"reveal":[180],"simple":[183],"implementation,":[184],"SeeAct-ATA,":[185],"does":[186],"not":[187],"perform":[188],"well":[189],"compared":[190],"more":[193,231],"advanced":[194],"PinATA":[195,206],"implementation":[196],"when":[197],"cases":[200],"(50%":[201],"performance":[202],"improvement).":[203],"However,":[204],"while":[205],"obtains":[207],"around":[208],"60%":[209],"correct":[211],"verdict":[212],"up":[214],"promising":[217],"94%":[218],"specificity,":[219],"several":[222],"be":[227],"addressed":[228],"develop":[230],"resilient":[232],"reliable":[234],"ATAs,":[235],"paving":[236],"way":[238],"robust,":[240],"low":[241],"automation.":[244]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":1}],"updated_date":"2026-08-14T07:06:38.338062","created_date":"2025-10-10T00:00:00"}