{"id":"https://openalex.org/W4414561387","doi":"https://doi.org/10.1145/3805712.3808592","title":"WebMall - A Multi-Shop Benchmark for Evaluating Web Agents","display_name":"WebMall - A Multi-Shop Benchmark for Evaluating Web Agents","publication_year":2026,"publication_date":"2026-07-10","ids":{"openalex":"https://openalex.org/W4414561387","doi":"https://doi.org/10.1145/3805712.3808592"},"language":"en","primary_location":{"id":"doi:10.1145/3805712.3808592","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3808592","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805712.3808592","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082603069","display_name":"Ralph Peeters","orcid":"https://orcid.org/0000-0003-3174-2616"},"institutions":[{"id":"https://openalex.org/I177802217","display_name":"University of Mannheim","ror":"https://ror.org/031bsb921","country_code":"DE","type":"education","lineage":["https://openalex.org/I177802217"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ralph Peeters","raw_affiliation_strings":["Data and Web Science Group, University of Mannheim, Mannheim, Germany"],"raw_orcid":"https://orcid.org/0000-0003-3174-2616","affiliations":[{"raw_affiliation_string":"Data and Web Science Group, University of Mannheim, Mannheim, Germany","institution_ids":["https://openalex.org/I177802217"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Aaron Steiner","orcid":"https://orcid.org/0009-0006-6946-7057"},"institutions":[{"id":"https://openalex.org/I177802217","display_name":"University of Mannheim","ror":"https://ror.org/031bsb921","country_code":"DE","type":"education","lineage":["https://openalex.org/I177802217"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Aaron Steiner","raw_affiliation_strings":["Data and Web Science Group, University of Mannheim, Mannheim, Germany"],"raw_orcid":"https://orcid.org/0009-0006-6946-7057","affiliations":[{"raw_affiliation_string":"Data and Web Science Group, University of Mannheim, Mannheim, Germany","institution_ids":["https://openalex.org/I177802217"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Luca Schwarz","orcid":"https://orcid.org/0009-0003-4751-4607"},"institutions":[{"id":"https://openalex.org/I177802217","display_name":"University of Mannheim","ror":"https://ror.org/031bsb921","country_code":"DE","type":"education","lineage":["https://openalex.org/I177802217"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Luca Schwarz","raw_affiliation_strings":["Data and Web Science Group, University of Mannheim, Mannheim, Germany"],"raw_orcid":"https://orcid.org/0009-0003-4751-4607","affiliations":[{"raw_affiliation_string":"Data and Web Science Group, University of Mannheim, Mannheim, Germany","institution_ids":["https://openalex.org/I177802217"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119747349","display_name":"Julian Yuya Caspary","orcid":null},"institutions":[{"id":"https://openalex.org/I177802217","display_name":"University of Mannheim","ror":"https://ror.org/031bsb921","country_code":"DE","type":"education","lineage":["https://openalex.org/I177802217"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Julian Yuya Caspary","raw_affiliation_strings":["Data and Web Science Group, University of Mannheim, Mannheim, Germany"],"raw_orcid":"https://orcid.org/0009-0001-0606-4115","affiliations":[{"raw_affiliation_string":"Data and Web Science Group, University of Mannheim, Mannheim, Germany","institution_ids":["https://openalex.org/I177802217"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076876024","display_name":"Christian Bizer","orcid":"https://orcid.org/0000-0003-2367-0237"},"institutions":[{"id":"https://openalex.org/I177802217","display_name":"University of Mannheim","ror":"https://ror.org/031bsb921","country_code":"DE","type":"education","lineage":["https://openalex.org/I177802217"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Christian Bizer","raw_affiliation_strings":["Data and Web Science Group, University of Mannheim, Mannheim, Germany"],"raw_orcid":"https://orcid.org/0000-0003-2367-0237","affiliations":[{"raw_affiliation_string":"Data and Web Science Group, University of Mannheim, Mannheim, Germany","institution_ids":["https://openalex.org/I177802217"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I177802217"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3342","last_page":"3348"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.9089999794960022,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.9089999794960022,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8485000133514404},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6718000173568726},{"id":"https://openalex.org/keywords/product","display_name":"Product (mathematics)","score":0.5717999935150146},{"id":"https://openalex.org/keywords/web-application","display_name":"Web application","score":0.4909999966621399},{"id":"https://openalex.org/keywords/online-and-offline","display_name":"Online and offline","score":0.4018999934196472},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.366100013256073},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.3294000029563904}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8485000133514404},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7260000109672546},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6718000173568726},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.5717999935150146},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.4909999966621399},{"id":"https://openalex.org/C2780102126","wikidata":"https://www.wikidata.org/wiki/Q10928179","display_name":"Online and offline","level":2,"score":0.4018999934196472},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3734000027179718},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.366100013256073},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.3294000029563904},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32510000467300415},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.3125999867916107},{"id":"https://openalex.org/C35578498","wikidata":"https://www.wikidata.org/wiki/Q193424","display_name":"Web service","level":2,"score":0.3005000054836273},{"id":"https://openalex.org/C19351080","wikidata":"https://www.wikidata.org/wiki/Q1395034","display_name":"New product development","level":2,"score":0.28130000829696655},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.2768999934196472},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2727999985218048},{"id":"https://openalex.org/C2780428219","wikidata":"https://www.wikidata.org/wiki/Q16952335","display_name":"Cover (algebra)","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C41550386","wikidata":"https://www.wikidata.org/wiki/Q529909","display_name":"Multi-agent system","level":2,"score":0.2556000053882599},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2551000118255615}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3805712.3808592","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3808592","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2508.13024","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2508.13024","pdf_url":"https://arxiv.org/pdf/2508.13024","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:arXiv.org:2508.13024","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2508.13024","pdf_url":"https://arxiv.org/pdf/2508.13024","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2508.13024","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2508.13024","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1145/3805712.3808592","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3808592","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"LLM-based":[0],"web":[1,9,33,141],"agents":[2,34,37,71,121,142,192,217],"have":[3],"the":[4,22,27,43,51,55,59,134,160,203,209,212,215,225],"potential":[5],"to":[6,38,72,122,174],"automate":[7],"long-running":[8],"tasks,":[10,76],"such":[11,80],"as":[12,81,182,184],"searching":[13],"for":[14,31,54,139,177],"products":[15,24],"in":[16,195,224],"multiple":[17,113],"e-shops":[18],"and":[19,64,84,119,171,202,231],"subsequently":[20],"ordering":[21],"cheapest":[23,228],"that":[25,69,111,193],"meet":[26],"user's":[28],"needs.":[29],"Benchmarks":[30],"evaluating":[32,140],"either":[35],"require":[36,70],"perform":[39,73,123],"tasks":[40,91,165],"online":[41,67],"using":[42,48,190],"live":[44],"Web":[45],"or":[46,179],"offline":[47,78,136],"simulated":[49,152],"environments,":[50],"latter":[52],"allowing":[53],"exact":[56],"reproduction":[57],"of":[58,150,199,211],"experimental":[60],"setup.":[61],"While":[62],"DeepShop":[63],"ShoppingComp":[65],"provide":[66],"benchmarks":[68,79],"challenging":[74,144],"shopping":[75,146],"existing":[77],"WebShop,":[82],"WebArena,":[83],"Mind2Web":[85],"cover":[86],"only":[87],"comparatively":[88],"simple":[89],"e-commerce":[90,109],"performed":[92],"against":[93],"a":[94,101],"single":[95,102],"shop":[96],"containing":[97,115],"product":[98,117,156,169,229,233],"data":[99,118,157],"from":[100,159,167],"source.":[103],"What":[104],"is":[105,107],"missing":[106],"an":[108],"benchmark":[110,138],"simulates":[112],"shops":[114,153],"heterogeneous":[116],"requires":[120],"complex":[124],"retrieval":[125],"tasks.":[126,147],"We":[127,187],"fill":[128],"this":[129],"gap":[130],"by":[131],"introducing":[132],"WebMall,":[133],"first":[135],"multi-shop":[137],"on":[143],"comparison":[145],"WebMall":[148,164,189],"consists":[149],"four":[151],"populated":[154],"with":[155,214],"extracted":[158],"Common":[161],"Crawl.":[162],"The":[163,206],"range":[166],"specific":[168],"searches":[170,176],"price":[172],"comparisons":[173],"advanced":[175],"complementary":[178],"substitute":[180],"products,":[181],"well":[183],"checkout":[185],"processes.":[186],"validate":[188],"eight":[191],"differ":[194],"observation":[196],"space,":[197],"availability":[198],"short-term":[200],"memory,":[201],"employed":[204],"LLM.":[205],"validation":[207],"highlights":[208],"difficulty":[210],"benchmark,":[213],"best-performing":[216],"achieving":[218],"task":[219,226],"completion":[220],"rates":[221],"below":[222],"65%":[223],"categories":[227],"search":[230],"vague":[232],"search.":[234]},"counts_by_year":[],"updated_date":"2026-07-25T09:21:30.201066","created_date":"2025-09-27T00:00:00"}