{"id":"https://openalex.org/W7163707883","doi":"https://doi.org/10.48550/arxiv.2606.06462","title":"Benchmark Everything Everywhere All at Once","display_name":"Benchmark Everything Everywhere All at Once","publication_year":2026,"publication_date":"2026-06-04","ids":{"openalex":"https://openalex.org/W7163707883","doi":"https://doi.org/10.48550/arxiv.2606.06462"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.06462","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.06462","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.06462","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120308641","display_name":"Shiyun Xiong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiong, Shiyun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138010683","display_name":"Dongming Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Dongming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138005642","display_name":"Peiwen Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Peiwen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021103579","display_name":"Yuang Ai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ai, Yuang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138016523","display_name":"Bokang Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Bokang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138005637","display_name":"Wencheng Han","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Wencheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137935758","display_name":"Xiao-Hui Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xiao-Hui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137923492","display_name":"Xiangyu Yue","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yue, Xiangyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1940000057220459,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1940000057220459,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.09889999777078629,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.07400000095367432,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.9506999850273132},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.6873000264167786},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5665000081062317},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.5182999968528748},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.42559999227523804}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.9506999850273132},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7427999973297119},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6873000264167786},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5665000081062317},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.5182999968528748},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.49399998784065247},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.42559999227523804},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.414000004529953},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3314000070095062},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3237000107765198},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.3231000006198883},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32260000705718994}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.06462","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.06462","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.06462","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.06462","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Benchmarks":[0],"are":[1],"fundamental":[2],"for":[3,65],"evaluating":[4],"and":[5,8,13,23,31,80,86,110,120,170,180],"advancing":[6],"LLMs":[7],"MLLMs":[9],"by":[10],"providing":[11],"standardized":[12],"explicit":[14],"measures":[15],"of":[16],"performance.":[17],"However,":[18],"their":[19,42],"construction":[20,74],"is":[21],"labor-intensive":[22],"hard":[24],"to":[25,83,96,164],"reuse,":[26],"raising":[27],"concerns":[28],"about":[29],"sustainability":[30],"scalability.":[32],"Moreover,":[33],"existing":[34],"benchmarks":[35,160],"often":[36],"quickly":[37],"reach":[38],"performance":[39],"saturation":[40],"after":[41],"release,":[43],"resulting":[44],"in":[45],"insufficient":[46],"discrimination":[47],"among":[48],"state-of-the-art":[49],"models.":[50],"To":[51,89],"address":[52],"these":[53],"challenges,":[54],"we":[55,93,140],"introduce":[56],"Benchmark":[57,91,124],"Agent,":[58,92],"a":[59],"fully":[60],"autonomous":[61],"agentic":[62],"system":[63],"designed":[64],"benchmark":[66,73,129],"building.":[67],"Our":[68],"framework":[69],"orchestrates":[70],"the":[71,165,177],"complete":[72],"pipeline,":[75],"from":[76],"user":[77],"query":[78],"analysis":[79],"subtask":[81],"design":[82],"data":[84],"annotation":[85],"quality":[87],"control.":[88],"assess":[90],"implement":[94],"it":[95],"produce":[97],"15":[98],"representative":[99],"benchmarks,":[100],"spanning":[101],"diverse":[102],"evaluation":[103],"scenarios,":[104],"including":[105,115,145],"text":[106],"understanding,":[107,109],"multimodal":[108],"domain-specific":[111,152],"reasoning.":[112],"Extensive":[113],"experiments,":[114],"human":[116,133],"evaluation,":[117,139],"LLM-as-a-judge":[118],"assessment,":[119],"consistency":[121],"checks,":[122],"demonstrate":[123],"Agent":[125],"can":[126,161],"generate":[127],"high-quality":[128],"samples":[130],"with":[131,150],"minimal":[132],"involvement.":[134],"More":[135],"importantly,":[136],"through":[137],"continual":[138],"observe":[141],"several":[142],"insightful":[143],"findings,":[144],"that":[146,157],"current":[147],"models":[148],"struggle":[149],"certain":[151],"reasoning":[153],"tasks.":[154],"We":[155],"believe":[156],"rapidly":[158],"evolving":[159],"contribute":[162],"significantly":[163],"research":[166],"community.":[167],"The":[168],"preview":[169],"code":[171,181],"will":[172],"be":[173],"publicly":[174],"available":[175],"at":[176],"demo":[178],"page":[179],"repository.":[182]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-06T00:00:00"}