{"id":"https://openalex.org/W4406092746","doi":"https://doi.org/10.48550/arxiv.2501.01540","title":"BoxingGym: Benchmarking Progress in Automated Experimental Design and Model Discovery","display_name":"BoxingGym: Benchmarking Progress in Automated Experimental Design and Model Discovery","publication_year":2025,"publication_date":"2025-01-02","ids":{"openalex":"https://openalex.org/W4406092746","doi":"https://doi.org/10.48550/arxiv.2501.01540"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2501.01540","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.01540","pdf_url":"https://arxiv.org/pdf/2501.01540","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2501.01540","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085625870","display_name":"Kanishk Gandhi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gandhi, Kanishk","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102821532","display_name":"Michael Yu Li","orcid":"https://orcid.org/0009-0001-5881-458X","observed_orcids":["https://orcid.org/0009-0001-5881-458X"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li, Michael Y.","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115780746","display_name":"Lyle Goodyear","orcid":"https://orcid.org/0009-0001-1926-2196","observed_orcids":["https://orcid.org/0009-0001-1926-2196"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Goodyear, Lyle","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Bhatia, Agam","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bhatia, Agam","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111411900","display_name":"L. Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li, Louise","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Bhaskar, Aditi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bhaskar, Aditi","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zaman, Mohammed","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zaman, Mohammed","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001961716","display_name":"Noah D. Goodman","orcid":"https://orcid.org/0000-0002-9176-8802","observed_orcids":["https://orcid.org/0000-0002-9176-8802"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Goodman, Noah D.","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.3163999915122986,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.3163999915122986,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.15299999713897705,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.06639999896287918,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/model-discovery","display_name":"model discovery","score":0.8610000014305115},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.6010000109672546},{"id":"https://openalex.org/keywords/generative-probabilistic-models","display_name":"probabilistic generative models","score":0.9100000262260437},{"id":"https://openalex.org/keywords/expected-information-gain","display_name":"expected information gain","score":0.9879999756813049},{"id":"https://openalex.org/keywords/gpt-4o","display_name":"GPT-4o","score":0.7860000133514404},{"id":"https://openalex.org/keywords/scientific-discovery","display_name":"scientific discovery","score":0.5199999809265137}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8920335173606873},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4894615411758423},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4714473485946655},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3706486225128174},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.1537005603313446},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.05160138010978699}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2501.01540","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.01540","pdf_url":"https://arxiv.org/pdf/2501.01540","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2501.01540","doi":"https://doi.org/10.48550/arxiv.2501.01540","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2501.01540","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2501.01540","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.01540","pdf_url":"https://arxiv.org/pdf/2501.01540","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4406092746.pdf","grobid_xml":"https://content.openalex.org/works/W4406092746.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7222995582","https://openalex.org/W7202183629","https://openalex.org/W7118539294","https://openalex.org/W6929217661","https://openalex.org/W7159970066","https://openalex.org/W7162751963","https://openalex.org/W4415318097","https://openalex.org/W7222478994","https://openalex.org/W7168272912","https://openalex.org/W7127605006"],"abstract_inverted_index":{"Understanding":[0],"the":[1,37,149,166,250],"world":[2],"and":[3,24,57,87,92,99,179,197,243],"explaining":[4],"it":[5],"with":[6,70,111,239,253],"scientific":[7,34,42,52,85,94,114,128,139,174,191,205],"theories":[8],"is":[9,176],"a":[10,68,84,107,113,138,169,177,190],"central":[11],"aspiration":[12],"of":[13,40,62,168],"artificial":[14],"intelligence":[15],"research.":[16],"Proposing":[17],"theories,":[18],"designing":[19],"experiments":[20],"to":[21,33,50,82,133,142,183,193,207,216],"test":[22,47,83],"them,":[23],"then":[25,198],"revising":[26,93],"them":[27,59],"based":[28],"on":[29],"data":[30,81],"are":[31,123],"fundamental":[32],"discovery.":[35,245],"Despite":[36],"significant":[38],"promise":[39],"LLM-based":[41,251],"agents,":[43],"no":[44],"benchmarks":[45],"systematically":[46,74],"LLM's":[48],"ability":[49,141],"propose":[51],"models,":[53],"collect":[54,143],"experimental":[55,77,145,241],"data,":[56,146],"revise":[58],"in":[60],"light":[61],"new":[63],"data.":[64],"We":[65,230,246],"introduce":[66],"BoxingGym,":[67],"benchmark":[69],"10":[71],"environments":[72],"for":[73],"evaluating":[75],"both":[76,240],"design":[78,242],"(e.g.":[79,90],"collecting":[80],"theory)":[86],"model":[88,110,186,196,223,244,257],"discovery":[89],"proposing":[91],"theories).":[95],"To":[96,135],"enable":[97],"tractable":[98],"quantitative":[100],"evaluation,":[101,219],"we":[102,147,188,220],"implement":[103],"each":[104],"environment":[105],"as":[106,227,236],"generative":[108,170],"probabilistic":[109,121],"which":[112,157],"agent":[115,192,206,252],"can":[116],"run":[117],"interactive":[118],"experiments.":[119],"These":[120],"models":[122],"drawn":[124],"from":[125,131],"various":[126],"real-world":[127],"domains":[129],"ranging":[130],"psychology":[132],"ecology.":[134],"quantitatively":[136,184],"evaluate":[137,185],"agent's":[140],"informative":[144],"compute":[148,221],"expected":[150],"information":[151],"gain":[152],"(EIG),":[153],"an":[154,161,254],"information-theoretic":[155],"quantity":[156],"measures":[158],"how":[159],"much":[160],"experiment":[162],"reduces":[163],"uncertainty":[164],"about":[165,211],"parameters":[167],"model.":[171],"A":[172],"good":[173],"theory":[175],"concise":[178],"predictive":[180],"explanation.":[181],"Therefore,":[182],"discovery,":[187],"ask":[189],"explain":[194],"their":[195],"assess":[199],"whether":[200],"this":[201,212,217],"explanation":[202],"enables":[203],"another":[204],"make":[208],"reliable":[209],"predictions":[210],"environment.":[213],"In":[214],"addition":[215],"explanation-based":[218],"standard":[222],"evaluation":[224],"metrics":[225],"such":[226,235],"prediction":[228],"errors.":[229],"find":[231,247],"that":[232,248],"current":[233],"LLMs,":[234],"GPT-4o,":[237],"struggle":[238],"augmenting":[249],"explicit":[255],"statistical":[256],"does":[258],"not":[259],"reliably":[260],"improve":[261],"these":[262],"results.":[263]},"counts_by_year":[],"updated_date":"2026-09-30T09:27:27.532063","created_date":"2025-01-07T00:00:00"}