{"id":"https://openalex.org/W7204493988","doi":"https://doi.org/10.48550/arxiv.2608.25286","title":"BixBench3: Benchmarking AI agents on research-study-scale computational biology tasks","display_name":"BixBench3: Benchmarking AI agents on research-study-scale computational biology tasks","publication_year":2026,"publication_date":"2026-08-26","ids":{"openalex":"https://openalex.org/W7204493988","doi":"https://doi.org/10.48550/arxiv.2608.25286"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.25286","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.25286","pdf_url":"https://arxiv.org/pdf/2608.25286","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.25286","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088682596","display_name":"Zane Koch","orcid":"https://orcid.org/0000-0003-2259-3065","observed_orcids":["https://orcid.org/0000-0003-2259-3065"]},"institutions":[{"id":"https://openalex.org/I4405259067","display_name":"Edison Scientific Inc. (United States)","ror":"https://ror.org/01p9amb16","country_code":"US","type":"company","lineage":["https://openalex.org/I4405259067"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Koch, Zane","raw_affiliation_strings":["Edison Scientific, Inc. , San Francisco , CA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Edison Scientific, Inc. , San Francisco , CA , USA","institution_ids":["https://openalex.org/I4405259067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089880023","display_name":"Asmamaw T. Wassie","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405259067","display_name":"Edison Scientific Inc. (United States)","ror":"https://ror.org/01p9amb16","country_code":"US","type":"company","lineage":["https://openalex.org/I4405259067"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wassie, Asmamaw T.","raw_affiliation_strings":["Edison Scientific, Inc. , San Francisco , CA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Edison Scientific, Inc. , San Francisco , CA , USA","institution_ids":["https://openalex.org/I4405259067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058865154","display_name":"Javier Vald\u00e9s-Alem\u00e1n","orcid":"https://orcid.org/0000-0002-1577-1057","observed_orcids":["https://orcid.org/0000-0002-1577-1057"]},"institutions":[{"id":"https://openalex.org/I4405259067","display_name":"Edison Scientific Inc. (United States)","ror":"https://ror.org/01p9amb16","country_code":"US","type":"company","lineage":["https://openalex.org/I4405259067"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Valdes-Aleman, Javier","raw_affiliation_strings":["Edison Scientific, Inc. , San Francisco , CA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Edison Scientific, Inc. , San Francisco , CA , USA","institution_ids":["https://openalex.org/I4405259067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141411013","display_name":"Jason Lee","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405259067","display_name":"Edison Scientific Inc. (United States)","ror":"https://ror.org/01p9amb16","country_code":"US","type":"company","lineage":["https://openalex.org/I4405259067"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lee, Jason","raw_affiliation_strings":["Edison Scientific, Inc. , San Francisco , CA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Edison Scientific, Inc. , San Francisco , CA , USA","institution_ids":["https://openalex.org/I4405259067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087065415","display_name":"Michaela M. Hinks","orcid":"https://orcid.org/0009-0005-7918-3544","observed_orcids":["https://orcid.org/0009-0005-7918-3544"]},"institutions":[{"id":"https://openalex.org/I4405259067","display_name":"Edison Scientific Inc. (United States)","ror":"https://ror.org/01p9amb16","country_code":"US","type":"company","lineage":["https://openalex.org/I4405259067"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hinks, Michaela M.","raw_affiliation_strings":["Edison Scientific, Inc. , San Francisco , CA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Edison Scientific, Inc. , San Francisco , CA , USA","institution_ids":["https://openalex.org/I4405259067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061828831","display_name":"Samuel G. Rodriques","orcid":"https://orcid.org/0000-0002-2509-0861","observed_orcids":["https://orcid.org/0000-0002-2509-0861"]},"institutions":[{"id":"https://openalex.org/I4405259067","display_name":"Edison Scientific Inc. (United States)","ror":"https://ror.org/01p9amb16","country_code":"US","type":"company","lineage":["https://openalex.org/I4405259067"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rodriques, Samuel G.","raw_affiliation_strings":["Edison Scientific, Inc. , San Francisco , CA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Edison Scientific, Inc. , San Francisco , CA , USA","institution_ids":["https://openalex.org/I4405259067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016077667","display_name":"Andrew D. White","orcid":"https://orcid.org/0000-0001-7869-0118","observed_orcids":["https://orcid.org/0000-0001-7869-0118","https://orcid.org/0000-0002-6647-3965"]},"institutions":[{"id":"https://openalex.org/I4405259067","display_name":"Edison Scientific Inc. (United States)","ror":"https://ror.org/01p9amb16","country_code":"US","type":"company","lineage":["https://openalex.org/I4405259067"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"White, Andrew D.","raw_affiliation_strings":["Edison Scientific, Inc. , San Francisco , CA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Edison Scientific, Inc. , San Francisco , CA , USA","institution_ids":["https://openalex.org/I4405259067"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031830117","display_name":"Jon M. Laurent","orcid":"https://orcid.org/0000-0001-6583-4741","observed_orcids":["https://orcid.org/0000-0001-6583-4741"]},"institutions":[{"id":"https://openalex.org/I4405259067","display_name":"Edison Scientific Inc. (United States)","ror":"https://ror.org/01p9amb16","country_code":"US","type":"company","lineage":["https://openalex.org/I4405259067"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Laurent, Jon M.","raw_affiliation_strings":["Edison Scientific, Inc. , San Francisco , CA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Edison Scientific, Inc. , San Francisco , CA , USA","institution_ids":["https://openalex.org/I4405259067"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4405259067"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.7764000296592712,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.7764000296592712,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.11079999804496765,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.03240000084042549,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ai-agents","display_name":"AI agents","score":0.7559999823570251},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8820000290870667},{"id":"https://openalex.org/keywords/computational-biology","display_name":"computational biology","score":0.7749999761581421},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.7940000295639038},{"id":"https://openalex.org/keywords/data-processing","display_name":"data processing","score":0.37400001287460327},{"id":"https://openalex.org/keywords/peak-calling","display_name":"peak calling","score":0.5440000295639038},{"id":"https://openalex.org/keywords/differential-gene-expression","display_name":"differential gene expression","score":0.9100000262260437},{"id":"https://openalex.org/keywords/token-efficiency","display_name":"token efficiency","score":0.5669999718666077},{"id":"https://openalex.org/keywords/domain-generalization","display_name":"domain generalization","score":0.4909999966621399}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7389000058174133},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7226999998092651},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.6396999955177307},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5763999819755554},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5317999720573425},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.492900013923645},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.41019999980926514},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.37560001015663147},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.3587999939918518},{"id":"https://openalex.org/C86532276","wikidata":"https://www.wikidata.org/wiki/Q1184065","display_name":"Delegation","level":2,"score":0.35190001130104065},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.33959999680519104},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3305000066757202},{"id":"https://openalex.org/C201797286","wikidata":"https://www.wikidata.org/wiki/Q4914986","display_name":"Biological data","level":2,"score":0.3091999888420105},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.2989000082015991},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.26109999418258667},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.25870001316070557},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.25286","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.25286","pdf_url":"https://arxiv.org/pdf/2608.25286","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.25286","doi":"https://doi.org/10.48550/arxiv.2608.25286","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.25286","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.25286","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.25286","pdf_url":"https://arxiv.org/pdf/2608.25286","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204493988.pdf","grobid_xml":"https://content.openalex.org/works/W7204493988.grobid-xml"},"referenced_works_count":44,"referenced_works":["https://openalex.org/W2313339984","https://openalex.org/W3118387479","https://openalex.org/W4390509917","https://openalex.org/W4391876619","https://openalex.org/W4394769065","https://openalex.org/W4398203645","https://openalex.org/W4399991282","https://openalex.org/W4402345509","https://openalex.org/W4403925918","https://openalex.org/W4412538787","https://openalex.org/W4412800263","https://openalex.org/W4412817705","https://openalex.org/W4412903737","https://openalex.org/W4413446803","https://openalex.org/W4414145382","https://openalex.org/W4414624754","https://openalex.org/W4415329539","https://openalex.org/W4415379685","https://openalex.org/W4415797693","https://openalex.org/W4415798813","https://openalex.org/W4415991135","https://openalex.org/W4416437085","https://openalex.org/W4417097161","https://openalex.org/W4417159298","https://openalex.org/W6892009760","https://openalex.org/W6948174282","https://openalex.org/W7118077060","https://openalex.org/W7118690722","https://openalex.org/W7119871385","https://openalex.org/W7125193990","https://openalex.org/W7127951739","https://openalex.org/W7127953859","https://openalex.org/W7134112184","https://openalex.org/W7134141374","https://openalex.org/W7147287373","https://openalex.org/W7152485245","https://openalex.org/W7154229203","https://openalex.org/W7161734976","https://openalex.org/W7162627714","https://openalex.org/W7164012646","https://openalex.org/W7166198555","https://openalex.org/W7166736473","https://openalex.org/W7167635513","https://openalex.org/W7167845123"],"related_works":["https://openalex.org/W6948174282","https://openalex.org/W7222754646","https://openalex.org/W7160715433","https://openalex.org/W7202253502","https://openalex.org/W7152485245","https://openalex.org/W7214743407","https://openalex.org/W7161178917","https://openalex.org/W7155372121","https://openalex.org/W4414028042","https://openalex.org/W7126207648"],"abstract_inverted_index":{"Artificial":[0],"intelligence":[1],"(AI)":[2],"promises":[3],"to":[4,18,48,54,61,70,88,118,180,234,274],"accelerate":[5],"biological":[6,51],"research":[7,28,77,98,121],"by":[8],"automating":[9],"computational":[10,21],"analyses.":[11],"Yet":[12],"the":[13,24,43,63,73,76,89,120,143,150,158,239,251],"ability":[14,273],"of":[15,26,45,65,85,116,160,286],"AI":[16,46],"agents":[17,47,225,253],"execute":[19,113,276],"on":[20,189,196,203,209],"biology":[22],"at":[23,216,221],"scale":[25],"complete":[27,235],"studies":[29],"has":[30],"not":[31],"been":[32],"systematically":[33],"evaluated.":[34],"Here":[35],"we":[36,164],"introduce":[37],"BixBench3,":[38],"a":[39,68,97,107,114],"benchmark":[40],"that":[41,166,267],"measures":[42],"capacity":[44],"process":[49],"raw":[50,103,193,287],"data":[52,104,124],"through":[53],"scientific":[55,109,293],"results.":[56],"We":[57],"designed":[58],"BixBench3":[59,155],"tasks":[60,156,190,197,204],"mirror":[62],"delegation":[64],"work":[66,291],"from":[67,106,127,173],"scientist":[69,74],"an":[71,94],"agent:":[72],"chooses":[75],"question":[78],"and":[79,102,111,147,208,232,248,257,289],"high-level":[80],"methods,":[81],"then":[82],"delegates":[83],"implementation":[84],"all":[86],"analyses":[87,117,210],"agent.":[90],"In":[91],"each":[92,236],"task,":[93,237],"agent":[95],"receives":[96],"objective,":[99],"methodological":[100],"guidance,":[101],"derived":[105],"published":[108],"study,":[110],"must":[112],"sequence":[115],"achieve":[119,170],"objective.":[122],"The":[123],"artifacts":[125,145],"resulting":[126],"these":[128],"analyses,":[129],"such":[130],"as":[131],"peak":[132],"call":[133],"matrices":[134],"or":[135],"differential":[136],"expression":[137],"tables,":[138],"are":[139],"programmatically":[140],"graded":[141],"against":[142],"corresponding":[144],"generated":[146],"reported":[148],"in":[149,271],"original":[151],"study.":[152],"Across":[153],"20":[154],"encompassing":[157],"generation":[159],"138":[161],"unique":[162],"artifacts,":[163],"find":[165],"13":[167],"frontier":[168],"models":[169],"scores":[171],"ranging":[172],"0.00":[174],"for":[175,182],"Gemini":[176],"3.1":[177],"Flash":[178],"Lite":[179],"0.48":[181],"GPT":[183],"5.6":[184],"Sol.":[185],"Agents":[186],"perform":[187],"worse":[188],"with":[191,198,205,238],"larger":[192],"datasets":[194],"(0.36":[195,215],"<100":[199],"GB":[200],"versus":[201],"0.10":[202],">100":[206],"GB)":[207],"requiring":[211],"more":[212],"sequential":[213,278],"steps":[214,218,280],"1-2":[217],"vs":[219],"0.24":[220],"3+).":[222],"On":[223],"average,":[224],"use":[226],"6.8":[227],"hours,":[228,244],"102":[229],"million":[230],"tokens,":[231,247],"\\$43":[233],"longest":[240],"attempts":[241],"consuming":[242],"24":[243],"1.07":[245],"billion":[246],"\\$525.":[249],"Notably,":[250],"highest-scoring":[252],"used":[254],"fewer":[255],"tokens":[256],"were":[258],"cheaper":[259],"than":[260],"less":[261],"performant":[262],"options.":[263],"These":[264],"results":[265],"reveal":[266],"LLMs":[268],"vary":[269],"substantially":[270],"their":[272],"(1)":[275],"multiple":[277],"analysis":[279],"coherently,":[281],"(2)":[282],"manage":[283],"large":[284],"quantities":[285],"data,":[288],"(3)":[290],"across":[292],"domains.":[294]},"counts_by_year":[{"year":2026,"cited_by_count":9}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-28T00:00:00"}