{"id":"https://openalex.org/W7204016057","doi":null,"title":"UltraHorizon: Benchmarking LLM-Agent Capabilities in Ultra Long-Horizon Scenarios","display_name":"UltraHorizon: Benchmarking LLM-Agent Capabilities in Ultra Long-Horizon Scenarios","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7204016057"},"language":"en","primary_location":{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-173629","doi":null,"is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-173629","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"},"volume":null,"type":"conference-paper","indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5147852783","display_name":"Haotian Luo","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Haotian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147840817","display_name":"Huaisong Zhang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Huaisong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147871491","display_name":"Xuelin ZHANG","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"ZHANG, Xuelin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147819715","display_name":"Haoyu Wang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Haoyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147873590","display_name":"Zeyu QIN","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"QIN, Zeyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147840179","display_name":"Wenjie Luo","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Wenjie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147866615","display_name":"Guozheng Ma","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Guozheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147814886","display_name":"Haiying He","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Haiying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147818105","display_name":"Yingsha Xie","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xie, Yingsha","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147854620","display_name":"Qiyang Zhou","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Qiyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147872484","display_name":"Zixuan Hu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Zixuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125568346","display_name":"Hongze Mi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mi, Hongze","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147814140","display_name":"Yibo Wang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yibo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020921636","display_name":"Naiqiang Tan","orcid":"https://orcid.org/0009-0008-4687-5212","observed_orcids":["https://orcid.org/0009-0008-4687-5212"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tan, Naiqiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147842936","display_name":"Hong CHEN","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"CHEN, Hong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029408111","display_name":"Yi R. Fung","orcid":"https://orcid.org/0009-0006-1869-0363","observed_orcids":["https://orcid.org/0009-0006-1869-0363"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"FUNG, Yi Ren","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147857688","display_name":"Chun Yuan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Chun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5147838685","display_name":"Shen Li","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Shen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.71164678,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9067000150680542,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9067000150680542,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.027000000700354576,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.013700000010430813,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.8880000114440918},{"id":"https://openalex.org/keywords/partially-observable-environments","display_name":"partially observable environments","score":0.6639999747276306},{"id":"https://openalex.org/keywords/exploration-tasks","display_name":"exploration tasks","score":0.6470000147819519},{"id":"https://openalex.org/keywords/planning","display_name":"planning","score":0.8859999775886536},{"id":"https://openalex.org/keywords/memory-management","display_name":"memory management","score":0.9419999718666077},{"id":"https://openalex.org/keywords/tool-use","display_name":"tool use","score":0.9800000190734863},{"id":"https://openalex.org/keywords/agent-evaluation","display_name":"agent evaluation","score":0.4020000100135803},{"id":"https://openalex.org/keywords/error-analysis","display_name":"error analysis","score":0.3889999985694885}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8345000147819519},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7283999919891357},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7172999978065491},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6194999814033508},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.5195000171661377},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5067999958992004},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.40939998626708984},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.37549999356269836},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.35269999504089355},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3325999975204468},{"id":"https://openalex.org/C2777655017","wikidata":"https://www.wikidata.org/wiki/Q1501161","display_name":"Toolbox","level":2,"score":0.3278999924659729},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31619998812675476},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.3043999969959259},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.30230000615119934},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.30000001192092896},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.29260000586509705},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2874000072479248},{"id":"https://openalex.org/C32848918","wikidata":"https://www.wikidata.org/wiki/Q845789","display_name":"Observable","level":2,"score":0.28220000863075256},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.28049999475479126},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.28040000796318054},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2775000035762787},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.27219998836517334},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.2694000005722046},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.25929999351501465},{"id":"https://openalex.org/C2776544517","wikidata":"https://www.wikidata.org/wiki/Q189447","display_name":"Unexpected events","level":2,"score":0.25920000672340393},{"id":"https://openalex.org/C44000306","wikidata":"https://www.wikidata.org/wiki/Q244330","display_name":"Hinge","level":2,"score":0.25429999828338623},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-173629","doi":null,"is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-173629","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4416830533","https://openalex.org/W7169855935","https://openalex.org/W7154478830","https://openalex.org/W7130734783","https://openalex.org/W4407368669","https://openalex.org/W7148248081","https://openalex.org/W7160943546","https://openalex.org/W7222041969","https://openalex.org/W7130675634","https://openalex.org/W7127888813"],"abstract_inverted_index":{"Autonomous":[0],"agents":[1,155],"have":[2],"recently":[3],"achieved":[4],"remarkable":[5],"progress,":[6],"yet":[7],"most":[8],"evaluations":[9],"focus":[10],"on":[11,41,148],"short-horizon,":[12],"fully":[13],"observable":[14,36],"tasks.":[15],"In":[16],"contrast,":[17],"many":[18],"real-world":[19],"tasks,":[20],"such":[21],"as":[22,81],"large-scale":[23],"software":[24],"development,":[25],"commercial":[26],"investment,":[27],"and":[28,34,47,113,116,129,141,212,221],"scientific":[29],"discovery,":[30],"unfold":[31],"in":[32,97,134,162,176,188],"long-horizon":[33,55,98,178],"partially":[35],"scenarios":[37],"where":[38,101],"success":[39],"hinges":[40],"sustained":[42,109],"reasoning,":[43,110],"planning,":[44,111],"memory":[45,112],"management,":[46,115],"tool":[48,131,146],"use.":[49],"Existing":[50],"benchmarks":[51],"rarely":[52],"capture":[53],"these":[54,91,163],"challenges.":[56,77],"To":[57,191],"bridge":[58],"this":[59],"gap,":[60],"we":[61,198],"introduce":[62],"UltraHorizon,":[63],"a":[64,82,173],"novel":[65],"benchmark":[66],"that":[67,154,184],"assesses":[68],"the":[69,121,194],"foundational":[70],"capabilities":[71],"required":[72],"to":[73,89,215],"address":[74],"complex":[75],"realworld":[76],"We":[78,181,206],"use":[79],"exploration":[80,179],"unifying":[83],"task":[84],"across":[85],"three":[86],"distinct":[87],"environments":[88],"validate":[90],"core":[92],"competencies.":[93],"Agents":[94],"are":[95],"designed":[96],"discovery":[99],"tasks":[100],"they":[102,137],"must":[103],"iteratively":[104],"uncover":[105],"hidden":[106],"rules":[107],"through":[108],"tools":[114],"interaction":[117],"with":[118],"environments.":[119],"Under":[120],"heaviest":[122],"scale":[123],"setting,":[124],"trajectories":[125],"average":[126],"200k+":[127],"tokens":[128],"400+":[130],"calls,":[132],"whereas":[133,165],"standard":[135],"configurations":[136],"still":[138],"exceed":[139],"35ktokens":[140],"involve":[142],"more":[143],"than":[144],"60":[145],"calls":[147],"average.":[149],"Our":[150],"extensive":[151],"experiments":[152],"reveal":[153],"powered":[156],"by":[157],"state-of-the-art":[158],"LLMs":[159],"consistently":[160],"underperform":[161],"settings,":[164],"human":[166],"participants":[167],"achieve":[168],"much":[169],"higher":[170],"scores,":[171],"underscoring":[172],"persistent":[174],"gap":[175],"agents\u2019":[177],"abilities.":[180],"also":[182],"observe":[183],"simple":[185],"scaling":[186],"fails":[187],"our":[189],"task.":[190],"better":[192],"illustrate":[193],"failure":[195],"of":[196,203,210],"agents,":[197],"conduct":[199],"an":[200],"in-depth":[201],"analysis":[202],"collected":[204],"trajectories.":[205],"identify":[207],"eight":[208],"types":[209],"errors":[211],"attribute":[213],"them":[214],"two":[216],"primary":[217],"causes:":[218],"in-context":[219],"locking":[220],"functional":[222],"fundamental":[223],"capability":[224],"gaps.":[225]},"counts_by_year":[],"updated_date":"2026-09-25T08:18:29.962427","created_date":"2026-08-23T00:00:00"}