{"id":"https://openalex.org/W7140197610","doi":"https://doi.org/10.48550/arxiv.2603.20807","title":"BenchBench: Benchmarking Automated Benchmark Generation","display_name":"BenchBench: Benchmarking Automated Benchmark Generation","publication_year":2026,"publication_date":"2026-03-21","ids":{"openalex":"https://openalex.org/W7140197610","doi":"https://doi.org/10.48550/arxiv.2603.20807"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.20807","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.20807","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.20807","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Zheng, Yandan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Yandan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Luo, Haoran","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Haoran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Lin, Zhenghong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Zhenghong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Wenjin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Wenjin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Tuan, Luu Anh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tuan, Luu Anh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4521999955177307,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4521999955177307,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.06480000168085098,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.05090000107884407,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8938999772071838},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7723000049591064},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6485999822616577},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6140999794006348},{"id":"https://openalex.org/keywords/fidelity","display_name":"Fidelity","score":0.5067999958992004},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.4767000079154968},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4577000141143799},{"id":"https://openalex.org/keywords/scheme","display_name":"Scheme (mathematics)","score":0.4153999984264374},{"id":"https://openalex.org/keywords/fuzz-testing","display_name":"Fuzz testing","score":0.3901999890804291},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.3862000107765198}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8938999772071838},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7723000049591064},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7480000257492065},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6485999822616577},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6140999794006348},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.5067999958992004},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4885999858379364},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.4767000079154968},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4577000141143799},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.4153999984264374},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4081999957561493},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3977999985218048},{"id":"https://openalex.org/C111065885","wikidata":"https://www.wikidata.org/wiki/Q1189053","display_name":"Fuzz testing","level":3,"score":0.3901999890804291},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.3862000107765198},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.35010001063346863},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.33799999952316284},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.32910001277923584},{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.3248000144958496},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.3215999901294708},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.30149999260902405},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.30149999260902405},{"id":"https://openalex.org/C2775936607","wikidata":"https://www.wikidata.org/wiki/Q466845","display_name":"Tracking (education)","level":2,"score":0.2985999882221222},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.2953000068664551},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.2897999882698059},{"id":"https://openalex.org/C2777561058","wikidata":"https://www.wikidata.org/wiki/Q2652119","display_name":"Program comprehension","level":4,"score":0.28949999809265137},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.27810001373291016},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.2761000096797943},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.2694000005722046},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.26649999618530273},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.2581999897956848},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.2538999915122986},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.251800000667572},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.2517000138759613},{"id":"https://openalex.org/C97686452","wikidata":"https://www.wikidata.org/wiki/Q7604153","display_name":"Static analysis","level":2,"score":0.250900000333786}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.20807","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.20807","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.20807","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.20807","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.5252814888954163}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Benchmarks":[0],"are":[1,26],"the":[2,179],"de":[3],"facto":[4],"standard":[5],"for":[6,74],"tracking":[7],"progress":[8],"in":[9],"large":[10],"language":[11],"models":[12,57,63],"(LLMs),":[13],"yet":[14],"static":[15],"test":[16],"sets":[17],"can":[18],"rapidly":[19],"saturate,":[20],"become":[21],"vulnerable":[22],"to":[23,28,60,92],"contamination,":[24],"and":[25,45,72,96,110,121,132,137,149,178,189],"costly":[27],"refresh.":[29],"Scalable":[30],"evaluation":[31,51],"of":[32,43,186],"open-ended":[33],"items":[34,99,147],"often":[35],"relies":[36],"on":[37],"LLM":[38],"judges,":[39],"introducing":[40],"additional":[41],"sources":[42],"bias":[44],"prompt":[46,88],"sensitivity.":[47],"We":[48,66],"argue":[49],"that":[50,157],"must":[52],"extend":[53],"beyond":[54],"how":[55,61],"well":[56,62],"answer":[58],"benchmarks":[59],"design":[64],"them.":[65],"introduce":[67],"BenchBench,":[68],"a":[69,101],"three-stage":[70],"pipeline":[71],"dataset":[73],"benchmarking":[75],"automated":[76],"benchmark":[77],"generation:":[78],"(i)":[79],"extract":[80],"structured":[81],"domain":[82],"cards":[83],"from":[84],"seed":[85],"benchmarks,":[86],"(ii)":[87],"multiple":[89],"designer":[90],"LLMs":[91],"generate":[93,141],"quota-controlled":[94],"suites,":[95],"(iii)":[97],"validate":[98],"with":[100,117,164,174],"multi-model":[102],"answerer":[103],"panel":[104],"using":[105],"exact/numeric/symbolic":[106],"verifiers":[107],"when":[108],"possible":[109],"rubric-guided":[111],"judging":[112],"otherwise,":[113],"yielding":[114],"designer--answerer":[115,181],"matrices":[116,182],"item-level":[118],"quality":[119],"flags":[120],"psychometric":[122],"diagnostics.":[123],"Across":[124],"nine":[125],"variants":[126],"spanning":[127],"computer":[128],"science,":[129],"mathematics,":[130],"medicine,":[131],"theory-of-mind":[133],"reasoning":[134],"(including":[135],"multilingual":[136],"multimodal":[138],"settings),":[139],"we":[140],"16.7K":[142],"items,":[143],"retain":[144],"~15K":[145],"core":[146],"post-filtering,":[148],"produce":[150],"~152K":[151],"graded":[152],"model--item":[153],"responses.":[154],"BenchBench":[155],"shows":[156],"benchmark-design":[158],"ability":[159],"is":[160,171,195],"only":[161],"moderately":[162],"correlated":[163],"answer-time":[165],"strength":[166],"(Spearman":[167],"rho":[168],"~0.37),":[169],"invalidity":[170],"negatively":[172],"associated":[173],"discrimination":[175],"(Pearson":[176],"r~0.62),":[177],"resulting":[180],"enable":[183],"scalable":[184],"audits":[185],"format/modality/language":[187],"fidelity":[188],"suite-dependent":[190],"self/family":[191],"interactions.":[192],"The":[193],"project":[194],"available":[196],"at:":[197],"https://github.com/koanatakiyo/BenchBench.":[198]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-03-25T00:00:00"}