{"id":"https://openalex.org/W7202278401","doi":"https://doi.org/10.18653/2026.emnlp-main.107","title":"DSAgentBench: Can Agents Automate End-to-End Data-Science Workflows in Real Computer Environments?","display_name":"DSAgentBench: Can Agents Automate End-to-End Data-Science Workflows in Real Computer Environments?","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7202278401","doi":"https://doi.org/10.18653/2026.emnlp-main.107"},"language":"en","primary_location":{"id":"doi:10.18653/2026.emnlp-main.107","doi":"https://doi.org/10.18653/2026.emnlp-main.107","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.107","pdf_url":"https://aclanthology.org/2026.emnlp-main.107.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://aclanthology.org/2026.emnlp-main.107.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136865855","display_name":"Mizanur Rahman","orcid":"https://orcid.org/0009-0008-2627-8664","observed_orcids":["https://orcid.org/0009-0008-2627-8664"]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mizanur Rahman","raw_affiliation_strings":["York University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147029778","display_name":"Mohammed Saidul Islam","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Mohammed Saidul Islam","raw_affiliation_strings":["Vector Institute $ Nanyang Technological University \u00b6 Salesforce AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Vector Institute $ Nanyang Technological University \u00b6 Salesforce AI Research","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029539188","display_name":"Ridwan Mahbub","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Ridwan Mahbub","raw_affiliation_strings":["York University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069546450","display_name":"Md Tahmid Rahman Laskar","orcid":"https://orcid.org/0009-0001-7880-299X","observed_orcids":["https://orcid.org/0009-0001-7880-299X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Md Tahmid Rahman Laskar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005443526","display_name":"Shafiq Joty","orcid":"https://orcid.org/0000-0002-9222-2641","observed_orcids":["https://orcid.org/0000-0002-9222-2641"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shafiq Joty","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5067722075","display_name":"Enamul Hoque","orcid":"https://orcid.org/0000-0002-9789-6645","observed_orcids":["https://orcid.org/0000-0002-9789-6645"]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Enamul Hoque Prince","raw_affiliation_strings":["York University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University","institution_ids":["https://openalex.org/I192455969"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.672,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.91719018,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"2517","last_page":"2540"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.5175999999046326,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.5175999999046326,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.1517000049352646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.04899999871850014,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.5370000004768372},{"id":"https://openalex.org/keywords/tool-orchestration","display_name":"tool orchestration","score":0.6840000152587891},{"id":"https://openalex.org/keywords/multi-step-reasoning","display_name":"multi-step reasoning","score":0.8569999933242798},{"id":"https://openalex.org/keywords/autonomous-agents","display_name":"autonomous agents","score":0.5099999904632568},{"id":"https://openalex.org/keywords/data-wrangling","display_name":"data wrangling","score":0.7170000076293945},{"id":"https://openalex.org/keywords/data-visualization","display_name":"data visualization","score":0.5099999904632568},{"id":"https://openalex.org/keywords/model-validation","display_name":"model validation","score":0.5239999890327454},{"id":"https://openalex.org/keywords/capability-gaps","display_name":"capability gaps","score":0.38499999046325684}],"concepts":[{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.8424999713897705},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7408000230789185},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6923999786376953},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.579200029373169},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3666999936103821},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3273000121116638},{"id":"https://openalex.org/C2776544517","wikidata":"https://www.wikidata.org/wiki/Q189447","display_name":"Unexpected events","level":2,"score":0.31310001015663147},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.27720001339912415},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.26809999346733093},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2667999863624573},{"id":"https://openalex.org/C13687954","wikidata":"https://www.wikidata.org/wiki/Q4826847","display_name":"Autonomous agent","level":2,"score":0.26260000467300415},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.26109999418258667},{"id":"https://openalex.org/C110593043","wikidata":"https://www.wikidata.org/wiki/Q7300787","display_name":"Real-time data","level":2,"score":0.25380000472068787},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2524999976158142},{"id":"https://openalex.org/C3020493868","wikidata":"https://www.wikidata.org/wiki/Q55631277","display_name":"Real world data","level":2,"score":0.2524000108242035}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/2026.emnlp-main.107","doi":"https://doi.org/10.18653/2026.emnlp-main.107","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.107","pdf_url":"https://aclanthology.org/2026.emnlp-main.107.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2608.10366","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.10366","pdf_url":"https://arxiv.org/pdf/2608.10366","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.10366","doi":"https://doi.org/10.48550/arxiv.2608.10366","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.10366","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/2026.emnlp-main.107","doi":"https://doi.org/10.18653/2026.emnlp-main.107","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.107","pdf_url":"https://aclanthology.org/2026.emnlp-main.107.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320310728","display_name":"Centre International de Recherche sur le Cancer","ror":"https://ror.org/00v452281"},{"id":"https://openalex.org/F4320314000","display_name":"Compute Canada","ror":"https://ror.org/03ty8yr27"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7202278401.pdf","grobid_xml":"https://content.openalex.org/works/W7202278401.grobid-xml"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W3177813494","https://openalex.org/W4285255856","https://openalex.org/W4327810158","https://openalex.org/W4385327683","https://openalex.org/W4387432648","https://openalex.org/W4389217618","https://openalex.org/W4398836283","https://openalex.org/W4401201100","https://openalex.org/W4403663202","https://openalex.org/W4404343203","https://openalex.org/W4404354143","https://openalex.org/W4404404752","https://openalex.org/W4404406411","https://openalex.org/W4404782852","https://openalex.org/W4406755521","https://openalex.org/W4415191234","https://openalex.org/W4416053353","https://openalex.org/W4416070493","https://openalex.org/W4416159271","https://openalex.org/W4416373998","https://openalex.org/W4416519583","https://openalex.org/W4416841435","https://openalex.org/W4417119060","https://openalex.org/W4417287263","https://openalex.org/W6948369276","https://openalex.org/W7077152995","https://openalex.org/W7077896124","https://openalex.org/W7125135101","https://openalex.org/W7197047016"],"related_works":["https://openalex.org/W7167280388","https://openalex.org/W4403663202","https://openalex.org/W7125224450","https://openalex.org/W7125713569","https://openalex.org/W7216191389","https://openalex.org/W4414920270","https://openalex.org/W4402671956","https://openalex.org/W4417098584","https://openalex.org/W4415795910","https://openalex.org/W4400484976"],"abstract_inverted_index":{"Real-world":[0],"data":[1,8],"science":[2],"involves":[3],"long-horizon":[4],"workflows":[5,49,77],"that":[6,116,137],"span":[7],"wrangling,":[9],"exploration,":[10],"modeling,":[11],"visualization,":[12],"and":[13,15,27,38,94,107,111,122,161,174],"validation,":[14],"require":[16],"coordinated":[17,108],"use":[18],"of":[19,61],"tools":[20],"such":[21],"as":[22,180],"notebooks,":[23],"IDEs,":[24],"terminals,":[25],"browsers,":[26],"databases":[28],"within":[29],"real":[30,79,175],"operating":[31],"environments.Yet":[32],"existing":[33],"benchmarks":[34],"lack":[35],"real-computer":[36],"interaction":[37],"do":[39],"not":[40],"evaluate":[41,70],"whether":[42,71],"agents":[43,72,150],"can":[44,73],"execute":[45],"complete":[46],"end-to-end":[47],"data-science":[48,62,89,176,188],"in":[50,98,104],"realistic":[51],"computing":[52],"environments,":[53],"failing":[54,155],"to":[55,69],"capture":[56],"the":[57,66,87,92,139],"multi-stage,":[58],"multi-tool":[59],"nature":[60],"practice.We":[63],"introduce":[64],"DSAGENTBENCH,":[65],"first":[67],"benchmark":[68],"automate":[74],"full":[75],"datascience":[76],"inside":[78],"computer":[80],"environments.DSAGENTBENCH":[81],"contains":[82],"275":[83],"diverse":[84],"tasks":[85],"covering":[86],"entire":[88],"life-cycle,":[90],"reflecting":[91],"complexity":[93],"tool":[95,109,157],"coordination":[96],"required":[97],"practice.Each":[99],"task":[100,145],"requires":[101],"grounding":[102],"decisions":[103],"intermediate":[105],"outputs":[106],"use,":[110],"includes":[112],"a":[113,166,181],"deterministic":[114],"evaluator":[115],"verifies":[117],"analytical":[118],"correctness,":[119],"visual":[120],"outputs,":[121],"model":[123],"performance":[124],"rather":[125],"than":[126],"code-only":[127],"execution.Our":[128],"extensive":[129],"experiments":[130],"with":[131],"15":[132],"closedand":[133],"open-source":[134],"models":[135],"show":[136],"even":[138],"strongest":[140],"agent,":[141],"Claude-4.6-Sonnet,achieves":[142],"only":[143],"56.70%":[144],"success,":[146],"while":[147],"all":[148],"opensource":[149],"remain":[151],"below":[152],"1%,":[153],"frequently":[154],"at":[156,192],"orchestration,":[158],"OS":[159],"grounding,":[160],"multi-step":[162],"reasoning.These":[163],"results":[164],"reveal":[165],"substantial":[167],"capability":[168],"gap":[169],"between":[170],"current":[171],"agentic":[172],"systems":[173],"workflows,":[177],"positioning":[178],"DSAGENTBENCH":[179,191],"foundation":[182],"for":[183],"developing":[184],"grounded,":[185],"verifiable,":[186],"autonomous":[187],"agents.We":[189],"release":[190],"https://github.":[193],"com/vis-nlp/DSAgentBench.":[194]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2026-08-13T00:00:00"}