{"id":"https://openalex.org/W7101416451","doi":"https://doi.org/10.48550/arxiv.2510.21652","title":"AstaBench: Rigorous Benchmarking of AI Agents with a Scientific Research Suite","display_name":"AstaBench: Rigorous Benchmarking of AI Agents with a Scientific Research Suite","publication_year":2025,"publication_date":"2025-10-24","ids":{"openalex":"https://openalex.org/W7101416451","doi":"https://doi.org/10.48550/arxiv.2510.21652"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2510.21652","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.21652","pdf_url":"https://arxiv.org/pdf/2510.21652","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.21652","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5038080713","display_name":"Jonathan Bragg","orcid":"https://orcid.org/0000-0001-5460-9047","observed_orcids":["https://orcid.org/0000-0001-5460-9047"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bragg, Jonathan","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033206229","display_name":"Mike D\u2019Arcy","orcid":"https://orcid.org/0000-0003-0355-7157","observed_orcids":["https://orcid.org/0000-0003-0355-7157","https://orcid.org/0000-0003-2280-917X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"D'Arcy, Mike","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091963084","display_name":"Nishant Balepur","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Balepur, Nishant","raw_affiliation_strings":["University of Maryland ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland ,","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005137956","display_name":"Dan Bareket","orcid":"https://orcid.org/0009-0008-2364-5676","observed_orcids":["https://orcid.org/0009-0008-2364-5676"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bareket, Dan","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007851620","display_name":"Bhavana Bharat Dalvi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dalvi, Bhavana","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000339878","display_name":"Sergey Feldman","orcid":"https://orcid.org/0000-0003-0386-0922","observed_orcids":["https://orcid.org/0000-0003-0386-0922"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Feldman, Sergey","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072172150","display_name":"Dany Haddad","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haddad, Dany","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Hwang, Jena D.","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hwang, Jena D.","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Jansen, Peter","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I138006243","display_name":"University of Arizona","ror":"https://ror.org/03m2x1q45","country_code":"US","type":"education","lineage":["https://openalex.org/I138006243"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jansen, Peter","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,","University of Arizona ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"University of Arizona ,","institution_ids":["https://openalex.org/I138006243"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Kishore, Varsha","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kishore, Varsha","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,","University of Washington ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"University of Washington ,","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069912951","display_name":"Bodhisattwa Prasad Majumder","orcid":"https://orcid.org/0009-0000-2691-305X","observed_orcids":["https://orcid.org/0009-0000-2691-305X","https://orcid.org/0000-0002-7026-3100"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Majumder, Bodhisattwa Prasad","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087743328","display_name":"Aakanksha Naik","orcid":"https://orcid.org/0000-0002-3673-0051","observed_orcids":["https://orcid.org/0000-0002-3673-0051"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Naik, Aakanksha","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104433152","display_name":"Sigal Rahamimov","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rahamimov, Sigal","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039359682","display_name":"Kyle Richardson","orcid":"https://orcid.org/0000-0003-4836-0753","observed_orcids":["https://orcid.org/0000-0003-4836-0753"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Richardson, Kyle","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130067916","display_name":"Amanpreet Singh","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Singh, Amanpreet","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084474211","display_name":"Harshit Surana","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Surana, Harshit","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078965793","display_name":"Aryeh Tiktinsky","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tiktinsky, Aryeh","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022594245","display_name":"Rosni Vasu","orcid":"https://orcid.org/0000-0002-0302-5065","observed_orcids":["https://orcid.org/0000-0002-0302-5065"]},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Vasu, Rosni","raw_affiliation_strings":["University of Zurich ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Zurich ,","institution_ids":["https://openalex.org/I202697423"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wiener, Guy","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wiener, Guy","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050044467","display_name":"Chloe Anastasiades","orcid":"https://orcid.org/0009-0009-5595-2299","observed_orcids":["https://orcid.org/0009-0009-5595-2299"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anastasiades, Chloe","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062428356","display_name":"Stefan Candra","orcid":"https://orcid.org/0009-0003-1940-3299","observed_orcids":["https://orcid.org/0009-0003-1940-3299"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Candra, Stefan","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003252084","display_name":"Jason Dunkelberger","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dunkelberger, Jason","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Emery, Dan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Emery, Dan","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011826930","display_name":"Rob Evans","orcid":"https://orcid.org/0009-0001-4549-2083","observed_orcids":["https://orcid.org/0009-0001-4549-2083"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Evans, Rob","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128632547","display_name":"Malachi Hamada","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hamada, Malachi","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062983431","display_name":"Regan Huff","orcid":"https://orcid.org/0009-0004-3658-5888","observed_orcids":["https://orcid.org/0009-0004-3658-5888"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Huff, Regan","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056105462","display_name":"Rodney Kinney","orcid":"https://orcid.org/0000-0002-3582-6537","observed_orcids":["https://orcid.org/0000-0002-3582-6537"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kinney, Rodney","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Latzke, Matt","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Latzke, Matt","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091455202","display_name":"Jaron Lochner","orcid":"https://orcid.org/0009-0009-1402-092X","observed_orcids":["https://orcid.org/0009-0009-1402-092X"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lochner, Jaron","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148794677","display_name":"Ruben Lozano Aguilera","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lozano-Aguilera, Ruben","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Nguyen, Cecile","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nguyen, Cecile","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Rao, Smita","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rao, Smita","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012552184","display_name":"Amber Tanaka","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tanaka, Amber","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127121862","display_name":"Brooke Vlahos","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vlahos, Brooke","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Clark, Peter","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Clark, Peter","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Downey, Doug","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Downey, Doug","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028476919","display_name":"Yoav Goldberg","orcid":"https://orcid.org/0000-0002-6497-829X","observed_orcids":["https://orcid.org/0000-0002-6497-829X"]},"institutions":[{"id":"https://openalex.org/I13955877","display_name":"Bar-Ilan University","ror":"https://ror.org/03kgsv495","country_code":"IL","type":"education","lineage":["https://openalex.org/I13955877"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["IL","US"],"is_corresponding":false,"raw_author_name":"Goldberg, Yoav","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,","Bar-Ilan University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"Bar-Ilan University ,","institution_ids":["https://openalex.org/I13955877"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077726785","display_name":"Ashish Sabharwal","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sabharwal, Ashish","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085011940","display_name":"Daniel S. Weld","orcid":"https://orcid.org/0000-0002-3255-0109","observed_orcids":["https://orcid.org/0000-0002-3255-0109"]},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Weld, Daniel S.","raw_affiliation_strings":["Asta Team , Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Asta Team , Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210140341"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.7681000232696533,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.7681000232696533,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.0544000007212162,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.04670000076293945,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ai-agents","display_name":"AI agents","score":0.921999990940094},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.8130000233650208},{"id":"https://openalex.org/keywords/artificial-intelligence-in-science","display_name":"artificial intelligence in science","score":0.382999986410141},{"id":"https://openalex.org/keywords/agent-evaluation","display_name":"agent evaluation","score":0.6370000243186951},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.6349999904632568},{"id":"https://openalex.org/keywords/search-tools","display_name":"search tools","score":0.5180000066757202},{"id":"https://openalex.org/keywords/baselines","display_name":"baselines","score":0.6399999856948853},{"id":"https://openalex.org/keywords/ai-scientists","display_name":"AI scientists","score":0.49399998784065247},{"id":"https://openalex.org/keywords/automated-literature-review","display_name":"literature review automation","score":0.5960000157356262},{"id":"https://openalex.org/keywords/experimental-replication","display_name":"experimental replication","score":0.49799999594688416}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8560000061988831},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.824400007724762},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7074999809265137},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5475999712944031},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5134000182151794},{"id":"https://openalex.org/C23213167","wikidata":"https://www.wikidata.org/wiki/Q2351730","display_name":"Scientific progress","level":2,"score":0.41999998688697815},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3783999979496002},{"id":"https://openalex.org/C2781083858","wikidata":"https://www.wikidata.org/wiki/Q17327049","display_name":"Scientific literature","level":2,"score":0.3666999936103821},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3375000059604645},{"id":"https://openalex.org/C2984917352","wikidata":"https://www.wikidata.org/wiki/Q12772819","display_name":"Scientific discovery","level":2,"score":0.30709999799728394},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2955000102519989},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.28949999809265137},{"id":"https://openalex.org/C114073186","wikidata":"https://www.wikidata.org/wiki/Q2631895","display_name":"Automated planning and scheduling","level":2,"score":0.2635999917984009},{"id":"https://openalex.org/C204983608","wikidata":"https://www.wikidata.org/wiki/Q2111958","display_name":"Productivity","level":2,"score":0.2632000148296356},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.2623000144958496},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2574999928474426}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2510.21652","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.21652","pdf_url":"https://arxiv.org/pdf/2510.21652","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.21652","doi":"https://doi.org/10.48550/arxiv.2510.21652","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.21652","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.21652","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.21652","pdf_url":"https://arxiv.org/pdf/2510.21652","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7101416451.pdf","grobid_xml":"https://content.openalex.org/works/W7101416451.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7164577541","https://openalex.org/W7220679195","https://openalex.org/W7128395166","https://openalex.org/W7158365662","https://openalex.org/W4403707291","https://openalex.org/W7164023999","https://openalex.org/W7166885752","https://openalex.org/W7118539294","https://openalex.org/W4414587939","https://openalex.org/W7117547917"],"abstract_inverted_index":{"AI":[0,43,251],"agents":[1,51,126,223,232],"hold":[2],"the":[3,167,193,256],"potential":[4],"to":[5,37,108,128,159,185],"revolutionize":[6],"scientific":[7,161,169,174,195],"productivity":[8],"by":[9,181],"automating":[10],"literature":[11],"reviews,":[12],"replicating":[13],"experiments,":[14],"analyzing":[15],"data,":[16],"and":[17,45,91,104,121,137,172,176,224],"even":[18],"proposing":[19],"new":[20],"directions":[21],"of":[22,49,76,113,156,217,221,230,258],"inquiry;":[23],"indeed,":[24],"there":[25],"are":[26],"now":[27],"many":[28,178],"such":[29,41,87,117],"agents,":[30,40],"ranging":[31],"from":[32,254],"general-purpose":[33],"\"deep":[34],"research\"":[35],"systems":[36],"specialized":[38],"science-specific":[39],"as":[42,88,118],"Scientist":[44],"AIGS.":[46],"Rigorous":[47],"evaluation":[48,229],"these":[50],"is":[52],"critical":[53],"for":[54,72,84,100,139,209],"progress.":[55],"Yet":[56],"existing":[57],"benchmarks":[58],"fall":[59],"short":[60],"on":[61,247],"several":[62,238],"fronts:":[63],"they":[64],"often":[65],"(1)":[66],"lack":[67,123],"reproducible":[68,205],"agent":[69,102,235],"tools":[70,201],"necessary":[71,127],"a":[73,149,153,214],"controlled":[74],"comparison":[75],"core":[77],"agentic":[78,157],"capabilities;":[79],"(2)":[80],"do":[81,95],"not":[82,96],"account":[83],"confounding":[85],"variables":[86],"model":[89],"cost":[90],"tool":[92],"access;":[93],"(3)":[94],"provide":[97,109,213],"standardized":[98],"interfaces":[99],"quick":[101],"prototyping":[103],"evaluation;":[105],"(4)":[106],"fail":[107],"holistic,":[110],"product-informed":[111],"measures":[112],"real-world":[114],"use":[115],"cases":[116],"science":[119,259],"research;":[120],"(5)":[122],"comprehensive":[124,215],"baseline":[125],"identify":[129],"true":[130],"advances.":[131],"In":[132],"response,":[133],"we":[134,146,212],"define":[135],"principles":[136],"tooling":[138],"more":[140],"rigorously":[141],"benchmarking":[142],"agents.":[143,188],"Using":[144],"these,":[145],"present":[147],"AstaBench,":[148],"suite":[150,190,216],"that":[151,202,243],"provides":[152],"holistic":[154],"measure":[155],"ability":[158],"perform":[160],"research,":[162],"comprising":[163],"2400+":[164],"problems":[165,179],"spanning":[166],"entire":[168],"discovery":[170],"process":[171],"multiple":[173],"domains,":[175],"including":[177],"inspired":[180],"actual":[182],"user":[183],"requests":[184],"deployed":[186],"Asta":[187,222],"Our":[189,227],"comes":[191],"with":[192,198],"first":[194],"research":[196,260],"environment":[197],"production-grade":[199],"search":[200],"enable":[203],"controlled,":[204],"evaluation,":[206],"better":[207],"accounting":[208],"confounders.":[210],"Alongside,":[211],"nine":[218],"science-optimized":[219],"classes":[220,236],"numerous":[225],"baselines.":[226],"extensive":[228],"57":[231],"across":[233],"22":[234],"reveals":[237],"interesting":[239],"findings,":[240],"most":[241],"importantly":[242],"despite":[244],"meaningful":[245],"progress":[246],"certain":[248],"individual":[249],"aspects,":[250],"remains":[252],"far":[253],"solving":[255],"challenge":[257],"assistance.":[261]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2025-10-28T00:00:00"}