{"id":"https://openalex.org/W4405254419","doi":"https://doi.org/10.48550/arxiv.2412.06745","title":"ONEBench to Test Them All: Sample-Level Benchmarking Over Open-Ended Capabilities","display_name":"ONEBench to Test Them All: Sample-Level Benchmarking Over Open-Ended Capabilities","publication_year":2024,"publication_date":"2024-12-09","ids":{"openalex":"https://openalex.org/W4405254419","doi":"https://doi.org/10.48550/arxiv.2412.06745"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2412.06745","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.06745","pdf_url":"https://arxiv.org/pdf/2412.06745","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2412.06745","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102645627","display_name":"Adhiraj Ghosh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ghosh, Adhiraj","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044199615","display_name":"Sebastian Dziadzio","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dziadzio, Sebastian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114202924","display_name":"Ameya Prabhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Prabhu, Ameya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108719787","display_name":"Vishaal Udandarao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Udandarao, Vishaal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102018867","display_name":"Samuel Albanie","orcid":"https://orcid.org/0000-0003-1732-9198"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Albanie, Samuel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5061457780","display_name":"Matthias Bethge","orcid":"https://orcid.org/0000-0002-6417-7812"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bethge, Matthias","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.3955000042915344,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.3955000042915344,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8423011302947998},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.670661985874176},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.649674117565155},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.44016233086586},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.38241708278656006},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.21508386731147766},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.21321073174476624},{"id":"https://openalex.org/keywords/marketing","display_name":"Marketing","score":0.11742466688156128},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.05999785661697388},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.04964551329612732}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8423011302947998},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.670661985874176},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.649674117565155},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.44016233086586},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.38241708278656006},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.21508386731147766},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.21321073174476624},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.11742466688156128},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.05999785661697388},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.04964551329612732},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2412.06745","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.06745","pdf_url":"https://arxiv.org/pdf/2412.06745","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2412.06745","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2412.06745","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2412.06745","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.06745","pdf_url":"https://arxiv.org/pdf/2412.06745","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7907426250","display_name":"EXC 2064: Maschinelles Lernen: Neue Perspektiven f\u00fcr die Wissenschaft","funder_award_id":"390727645","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G8915364236","display_name":null,"funder_award_id":"2064/1","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"}],"funders":[{"id":"https://openalex.org/F4320315389","display_name":"Open Philanthropy Project","ror":"https://ror.org/004d1k391"},{"id":"https://openalex.org/F4320317967","display_name":"International Max Planck Research School for Environmental, Cellular and Molecular Microbiology","ror":null},{"id":"https://openalex.org/F4320319740","display_name":"Good Ventures Foundation","ror":null},{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320321114","display_name":"Bundesministerium f\u00fcr Bildung und Forschung","ror":"https://ror.org/04pz7b180"},{"id":"https://openalex.org/F4320327553","display_name":"International Max Planck Research School for Advanced Methods in Process and Systems Engineering","ror":null}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4405254419.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W2291782699"],"abstract_inverted_index":{"Traditional":[0],"fixed":[1],"test":[2,57,72],"sets":[3],"fall":[4],"short":[5],"in":[6,198],"evaluating":[7],"open-ended":[8,41,223],"capabilities":[9,50,65],"of":[10,51,63,90,185],"foundation":[11,242],"models.":[12,243],"To":[13,127],"address":[14,128],"this,":[15],"we":[16,131,163,218],"propose":[17],"ONEBench(OpeN-Ended":[18],"Benchmarking),":[19],"a":[20,30,87,220,236],"new":[21],"testing":[22],"paradigm":[23],"that":[24,170],"consolidates":[25],"individual":[26],"evaluation":[27,42,85,189],"datasets":[28],"into":[29,138],"unified,":[31],"ever-expanding":[32],"sample":[33],"pool.":[34],"ONEBench":[35,59,102],"allows":[36],"users":[37],"to":[38,48,101,111,134,183,193,233],"generate":[39],"custom,":[40],"benchmarks":[43,100],"from":[44,98],"this":[45],"pool,":[46],"corresponding":[47],"specific":[49],"interest.":[52],"By":[53],"aggregating":[54,93],"samples":[55],"across":[56,214],"sets,":[58,73],"enables":[60],"the":[61,70,112,239],"assessment":[62],"diverse":[64,115],"beyond":[66],"those":[67,174],"covered":[68],"by":[69,176,191],"original":[71],"while":[74,117],"mitigating":[75],"overfitting":[76],"and":[77,92,107,150,207],"dataset":[78],"bias.":[79],"Most":[80],"importantly,":[81],"it":[82],"frames":[83],"model":[84,140,155,199],"as":[86],"collective":[88],"process":[89],"selecting":[91],"sample-level":[94,231],"tests.":[95],"The":[96],"shift":[97],"task-specific":[99],"introduces":[103],"two":[104],"challenges:":[105],"(1)heterogeneity":[106],"(2)incompleteness.":[108],"Heterogeneity":[109],"refers":[110],"aggregation":[113,143,166],"over":[114,228],"metrics,":[116],"incompleteness":[118],"describes":[119],"comparing":[120],"models":[121,206],"evaluated":[122],"on":[123],"different":[124],"data":[125],"subsets.":[126],"these":[129,215],"challenges,":[130],"explore":[132],"algorithms":[133],"aggregate":[135,227],"sparse":[136],"measurements":[137,186,232],"reliable":[139],"scores.":[141,178],"Our":[142],"algorithm":[144,167],"ensures":[145],"identifiability(asymptotically":[146],"recovering":[147],"ground-truth":[148],"scores)":[149],"rapid":[151],"convergence,":[152],"enabling":[153],"accurate":[154],"ranking":[156],"with":[157,173,195],"less":[158],"data.":[159],"On":[160],"homogenous":[161],"datasets,":[162],"show":[164],"our":[165],"provides":[168],"rankings":[169],"highly":[171],"correlate":[172],"produced":[175],"average":[177],"We":[179,201],"also":[180],"demonstrate":[181],"robustness":[182],"~95%":[184],"missing,":[187],"reducing":[188],"cost":[190],"up":[192],"20x":[194],"little-to-no":[196],"change":[197],"rankings.":[200],"introduce":[202],"ONEBench-LLM":[203],"for":[204,209,222],"language":[205],"ONEBench-LMM":[208],"vision-language":[210],"models,":[211],"unifying":[212],"evaluations":[213],"domains.":[216],"Overall,":[217],"present":[219],"technique":[221],"evaluation,":[224],"which":[225],"can":[226],"incomplete,":[229],"heterogeneous":[230],"continually":[234],"grow":[235],"benchmark":[237],"alongside":[238],"rapidly":[240],"developing":[241]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}