{"id":"https://openalex.org/W7157878540","doi":"https://doi.org/10.48550/arxiv.2604.24964","title":"Odysseys: Benchmarking Web Agents on Realistic Long Horizon Tasks","display_name":"Odysseys: Benchmarking Web Agents on Realistic Long Horizon Tasks","publication_year":2026,"publication_date":"2026-04-27","ids":{"openalex":"https://openalex.org/W7157878540","doi":"https://doi.org/10.48550/arxiv.2604.24964"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.24964","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.24964","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.24964","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102609431","display_name":"Lawrence Jang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jang, Lawrence Keunho","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009550511","display_name":"Jing Yu Koh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Koh, Jing Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134836629","display_name":"Daniel Fried","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fried, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134859956","display_name":"Ruslan Salakhutdinov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Salakhutdinov, Ruslan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.36730000376701355,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.36730000376701355,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09960000216960907,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.08309999853372574,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8237000107765198},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7127000093460083},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5983999967575073},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.597100019454956},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5856999754905701},{"id":"https://openalex.org/keywords/frontier","display_name":"Frontier","score":0.40130001306533813},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.3885999917984009}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8237000107765198},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.71670001745224},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7127000093460083},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5983999967575073},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.597100019454956},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5856999754905701},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40380001068115234},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.40130001306533813},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.3885999917984009},{"id":"https://openalex.org/C28761237","wikidata":"https://www.wikidata.org/wiki/Q7805321","display_name":"Time horizon","level":2,"score":0.3637000024318695},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.32499998807907104},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3197999894618988},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.3163999915122986},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3149999976158142},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.27869999408721924},{"id":"https://openalex.org/C13743948","wikidata":"https://www.wikidata.org/wiki/Q45842","display_name":"Web crawler","level":2,"score":0.26919999718666077},{"id":"https://openalex.org/C51485801","wikidata":"https://www.wikidata.org/wiki/Q16966861","display_name":"Efficient frontier","level":3,"score":0.2678000032901764},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.26510000228881836},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.2597000002861023},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2540999948978424}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.24964","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.24964","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.24964","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.24964","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Existing":[0],"web":[1,21,29,76],"agent":[2],"benchmarks":[3],"have":[4],"largely":[5],"converged":[6],"on":[7,85],"short,":[8],"single-site":[9],"tasks":[10,77],"that":[11,91,118,146,168,189,202,231],"frontier":[12,142,191],"models":[13,143,149],"are":[14],"approaching":[15],"saturation":[16],"on.":[17],"However,":[18],"real":[19,80],"world":[20,81],"use":[22],"consists":[23],"of":[24,60,73,112,154,215],"long-horizon,":[25],"multi-site":[26],"workflows.":[27],"Common":[28],"navigation":[30],"tasks,":[31,241],"such":[32,66],"as":[33],"comparing":[34],"products":[35],"across":[36,41],"different":[37],"domains,":[38],"planning":[39],"trips":[40],"multiple":[42,48],"services,":[43],"or":[44],"summarizing":[45],"information":[46],"from":[47,79],"search":[49],"queries,":[50],"require":[51],"sustained":[52],"context":[53],"and":[54,64,100,125,144,187,206,244],"cross-site":[55],"reasoning":[56],"over":[57],"potentially":[58,233],"hours":[59],"browsing.":[61],"To":[62],"capture":[63],"evaluate":[65],"behaviors,":[67],"we":[68,166],"introduce":[69,101,178],"Odysseys:":[70],"a":[71,102,127,151,171,179,222],"benchmark":[72,224],"200":[74],"long-horizon":[75,98,175,216],"derived":[78],"browsing":[82],"sessions":[83],"evaluated":[84],"the":[86,147,212],"live":[87],"Internet.":[88],"We":[89,116,138,177,238],"find":[90,145,188],"binary":[92],"pass/fail":[93],"evaluation":[94,136,214,242],"is":[95,170],"inadequate":[96],"for":[97,160,174,200,236],"settings":[99],"rubric-based":[103],"evaluation,":[104],"annotating":[105],"each":[106],"Odysseys":[107,210],"task":[108,164],"with":[109,123],"an":[110,197],"average":[111],"6.1":[113],"graded":[114],"rubrics.":[115],"demonstrate":[117],"this":[119],"yields":[120],"higher":[121],"agreement":[122],"humans":[124],"provides":[126],"more":[128],"fine-grained":[129],"signal":[130],"than":[131],"commonly":[132],"used":[133],"trajectory-level":[134],"LLM-as-a-judge":[135],"metrics.":[137],"tested":[139],"several":[140],"leading":[141],"strongest":[148],"achieve":[150,193],"success":[152],"rate":[153],"44.5%,":[155],"which":[156],"leaves":[157],"substantial":[158],"room":[159],"future":[161],"improvements.":[162],"Beyond":[163],"success,":[165],"argue":[167],"efficiency":[169],"first-class":[172],"concern":[173],"agents.":[176],"Trajectory":[180],"Efficiency":[181],"metric":[182],"(rubric":[183],"score":[184],"per":[185],"step)":[186],"even":[190],"agents":[192,201,230],"only":[194],"1.15%,":[195],"marking":[196],"evident":[198],"need":[199],"can":[203,232],"succeed":[204],"efficiently":[205],"not":[207],"simply":[208],"eventually.":[209],"isolates":[211],"critical":[213],"proficiency":[217],"in":[218],"open-web":[219],"environments,":[220],"providing":[221],"realistic":[223],"to":[225],"measure":[226],"progress":[227],"towards":[228],"computer-use":[229],"productively":[234],"operate":[235],"hours.":[237],"release":[239],"our":[240],"scripts,":[243],"other":[245],"results":[246],"at":[247],"https://odysseys-website.pages.dev":[248]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-30T00:00:00"}