{"id":"https://openalex.org/W7162067644","doi":"https://doi.org/10.48550/arxiv.2605.22535","title":"TerminalWorld: Benchmarking Agents on Real-World Terminal Tasks","display_name":"TerminalWorld: Benchmarking Agents on Real-World Terminal Tasks","publication_year":2026,"publication_date":"2026-05-21","ids":{"openalex":"https://openalex.org/W7162067644","doi":"https://doi.org/10.48550/arxiv.2605.22535"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.22535","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22535","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.22535","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136758018","display_name":"Zhaoyang Chu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chu, Zhaoyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136806768","display_name":"Jiarui Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Jiarui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107233117","display_name":"Xingyu Jiang","orcid":"https://orcid.org/0009-0006-0282-622X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Xingyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125252470","display_name":"Pengyu Zou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zou, Pengyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136804810","display_name":"Han Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Han","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136796631","display_name":"Chao Peng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Chao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136800370","display_name":"Peter O'Hearn","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"O'Hearn, Peter","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136746310","display_name":"Earl T. Barr","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barr, Earl T.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136803453","display_name":"Mark Harman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Harman, Mark","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012165852","display_name":"Federica Sarro","orcid":"https://orcid.org/0000-0002-9146-442X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sarro, Federica","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5101610258","display_name":"He Ye","orcid":"https://orcid.org/0000-0003-4807-2110"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ye, He","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.1800999939441681,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.1800999939441681,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.11919999867677689,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.10289999842643738,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8702999949455261},{"id":"https://openalex.org/keywords/terminal","display_name":"Terminal (telecommunication)","score":0.801800012588501},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7511000037193298},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.677299976348877},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.663100004196167},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4528000056743622},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.3411000072956085}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8702999949455261},{"id":"https://openalex.org/C2779664074","wikidata":"https://www.wikidata.org/wiki/Q3518405","display_name":"Terminal (telecommunication)","level":2,"score":0.801800012588501},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7511000037193298},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.684499979019165},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.677299976348877},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.663100004196167},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4528000056743622},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.38510000705718994},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.3411000072956085},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.3319000005722046},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.3260999917984009},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.30799999833106995},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.30660000443458557},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.30570000410079956},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.28040000796318054},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.2703000009059906},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.26919999718666077},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2644999921321869},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.26089999079704285},{"id":"https://openalex.org/C138827492","wikidata":"https://www.wikidata.org/wiki/Q6661985","display_name":"Data processing","level":2,"score":0.25949999690055847},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.250900000333786}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.22535","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22535","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.22535","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22535","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.49012917280197144}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0],"introduce":[1],"TerminalWorld,":[2],"a":[3,24,54,85,107],"scalable":[4,122],"data":[5],"engine":[6,22,117],"that":[7,75],"automatically":[8],"reverse-engineers":[9],"high-fidelity":[10],"evaluation":[11],"tasks":[12],"from":[13,36,99],"\"in-the-wild\"":[14],"terminal":[15,19,82,96,132],"recordings.":[16],"Processing":[17],"80,870":[18],"recordings,":[20],"the":[21],"yields":[23],"full":[25],"benchmark":[26],"of":[27,57,89],"1,530":[28],"validated":[29],"tasks,":[30],"spanning":[31],"18":[32],"real-world":[33,95,131],"categories,":[34],"ranging":[35],"short":[37],"everyday":[38],"operations":[39],"to":[40,110,127],"workflows":[41],"exceeding":[42],"50":[43],"steps,":[44],"and":[45,71,121,139],"covering":[46],"1,280":[47],"unique":[48],"commands.":[49],"From":[50],"these,":[51],"we":[52],"curate":[53],"Verified":[55],"subset":[56],"200":[58],"representative,":[59],"manually":[60],"reviewed":[61],"tasks.":[62],"Comprehensive":[63],"benchmarking":[64],"on":[65],"TerminalWorld-Verified":[66],"across":[67],"eight":[68],"frontier":[69],"models":[70],"six":[72],"agents":[73,129],"reveals":[74],"current":[76],"systems":[77],"still":[78],"struggle":[79],"with":[80,105],"authentic":[81,120],"workflows,":[83],"achieving":[84],"maximum":[86],"pass":[87],"rate":[88],"only":[90,106],"62.5%.":[91],"Moreover,":[92],"TerminalWorld":[93,119],"captures":[94],"capabilities":[97],"distinct":[98],"existing":[100],"expert-curated":[101],"benchmarks":[102],"(e.g.,":[103],"Terminal-Bench),":[104],"weak":[108],"correlation":[109],"their":[111],"scores":[112],"(Pearson":[113],"r=0.20).":[114],"The":[115],"automated":[116],"makes":[118],"by":[123],"construction,":[124],"enabling":[125],"it":[126],"evaluate":[128],"in":[130],"environments":[133],"as":[134],"developer":[135],"practices":[136],"evolve.":[137],"Data":[138],"code":[140],"are":[141],"available":[142],"at":[143],"https://github.com/EuniAI/TerminalWorld.":[144]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-23T00:00:00"}