{"id":"https://openalex.org/W7164842870","doi":"https://doi.org/10.48550/arxiv.2606.14397","title":"Running the Gauntlet: Re-evaluating the Capabilities of Agents Beyond Familiar Environments","display_name":"Running the Gauntlet: Re-evaluating the Capabilities of Agents Beyond Familiar Environments","publication_year":2026,"publication_date":"2026-06-12","ids":{"openalex":"https://openalex.org/W7164842870","doi":"https://doi.org/10.48550/arxiv.2606.14397"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.14397","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.14397","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.14397","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5122181345","display_name":"Mykola Vysotskyi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vysotskyi, Mykola","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138636016","display_name":"Runqi Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Runqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037079460","display_name":"Grzegorz Biziel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Biziel, Grzegorz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138678964","display_name":"Michal Zakrzewski","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zakrzewski, Michal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092429219","display_name":"Sebasti\u00e1n Montagna","orcid":"https://orcid.org/0009-0002-9910-9067"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Montagna, Sebastian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138648970","display_name":"Damian Rynczak","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rynczak, Damian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138655507","display_name":"Shreyansh Padarha","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Padarha, Shreyansh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138636584","display_name":"Kumail Alhamoud","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alhamoud, Kumail","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138640059","display_name":"Zihao Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Zihao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120320027","display_name":"William Lugoloobi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lugoloobi, William","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136418640","display_name":"Kai Rawal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rawal, Kai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5117356355","display_name":"Hanna Yershova","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yershova, Hanna","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138631193","display_name":"Xander Davies","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Davies, Xander","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086019258","display_name":"Taras Rumezhak","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rumezhak, Taras","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138658585","display_name":"Guohao Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Guohao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138672667","display_name":"Fazl Barez","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barez, Fazl","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138659661","display_name":"Baoyuan Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Baoyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121638617","display_name":"Arkadiusz Drohomirecki","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Drohomirecki, Arkadiusz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029186201","display_name":"Yarin Gal","orcid":"https://orcid.org/0000-0002-2733-2078"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gal, Yarin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138658731","display_name":"Chris Russell","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Russell, Chris","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138659211","display_name":"Christopher Summerfield","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Summerfield, Christopher","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055914149","display_name":"Adam Mahdi","orcid":"https://orcid.org/0000-0002-2329-4457"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mahdi, Adam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030628566","display_name":"Volodymyr Karpiv","orcid":"https://orcid.org/0009-0003-8439-8043"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karpiv, Volodymyr","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138676623","display_name":"Philip Torr","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Torr, Philip","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5091670711","display_name":"Adel Bibi","orcid":"https://orcid.org/0000-0002-6169-3918"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bibi, Adel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5993000268936157,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5993000268936157,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.1339000016450882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.03620000183582306,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7074999809265137},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6525999903678894},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.616100013256073},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.588100016117096},{"id":"https://openalex.org/keywords/modular-design","display_name":"Modular design","score":0.5723999738693237},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5503000020980835},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5260000228881836},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.3407999873161316}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7074999809265137},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6700999736785889},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6525999903678894},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.616100013256073},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.588100016117096},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.5723999738693237},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5503000020980835},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5260000228881836},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.37059998512268066},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.3407999873161316},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3305000066757202},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.33009999990463257},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3296999931335449},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32120001316070557},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.310699999332428},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.30390000343322754},{"id":"https://openalex.org/C13687954","wikidata":"https://www.wikidata.org/wiki/Q4826847","display_name":"Autonomous agent","level":2,"score":0.29829999804496765},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.2962000072002411},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.28290000557899475},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.2572000026702881}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.14397","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.14397","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.14397","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.14397","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"agentic":[1,157],"systems":[2,158],"continue":[3],"to":[4,18,57,148],"evolve":[5],"and":[6,36,55,86,102,127,139,185,210],"are":[7,26],"widely":[8],"deployed":[9],"in":[10,49,74,111,181],"real-world":[11,215],"scenarios,":[12,76],"there":[13],"is":[14],"a":[15,39,67,116,131,135,171],"growing":[16],"demand":[17],"faithfully":[19],"evaluate":[20],"their":[21,59],"capabilities.":[22],"However,":[23],"current":[24,207],"benchmarks":[25],"typically":[27],"built":[28],"on":[29,38,52,78,175,196],"popular":[30],"applications":[31,93],"with":[32,106,124,144],"relatively":[33],"simple":[34],"tasks":[35,109],"focus":[37],"narrow":[40],"set":[41],"of":[42],"capabilities":[43,81,184,209],"while":[44],"overlooking":[45],"broader":[46],"dimensions,":[47],"resulting":[48],"saturated":[50],"performance":[51],"modern":[53],"agents":[54],"failing":[56],"probe":[58],"limitations.":[60],"To":[61],"this":[62],"end,":[63],"we":[64],"introduce":[65],"GauntletBench,":[66,177],"web-based":[68,133],"benchmark":[69,114],"for":[70,213],"evaluating":[71],"agent":[72,129,168,208],"generalisation":[73],"challenging":[75,198],"focusing":[77],"three":[79],"underexplored":[80],"(temporal":[82],"perception,":[83],"graphical":[84],"understanding,":[85],"3D":[87,98],"reasoning),":[88],"across":[89],"five":[90],"less-covered":[91],"professional":[92],"(Video":[94],"Editor,":[95],"Workflow":[96],"Builder,":[97],"Modeller,":[99],"Flight":[100],"Analyser,":[101],"Circuit":[103],"Designer),":[104],"each":[105],"20":[107],"vision-intensive":[108],"(100":[110],"total).":[112],"Our":[113],"provides":[115],"modular":[117],"pipeline":[118],"that":[119,155],"comprises":[120],"an":[121,140],"environment":[122],"compatible":[123],"both":[125],"open-":[126],"closed-source":[128],"frameworks,":[130],"controlled":[132],"application,":[134],"well-structured":[136],"task":[137],"suite,":[138],"automated":[141],"evaluation":[142],"engine":[143],"diverse":[145],"metrics.":[146],"Contrary":[147],"widespread":[149],"expectations,":[150],"our":[151,176,197],"empirical":[152],"results":[153],"reveal":[154],"frontier":[156],"remain":[159],"far":[160],"from":[161],"achieving":[162],"human-level":[163],"performance.":[164],"Even":[165],"the":[166,179,203],"state-of-the-art":[167],"achieves":[169],"only":[170],"19.1%":[172],"success":[173,195],"rate":[174],"highlighting":[178],"limitations":[180],"these":[182],"overlooked":[183],"generalisation.":[186],"By":[187],"comparison,":[188],"non-expert":[189],"human":[190],"annotators":[191],"achieve":[192],"over":[193],"80%":[194],"yet":[199],"feasible":[200],"tasks,":[201],"revealing":[202],"substantial":[204],"gap":[205],"between":[206],"those":[211],"required":[212],"complex":[214],"scenarios.":[216]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-16T00:00:00"}