{"id":"https://openalex.org/W7161287961","doi":"https://doi.org/10.48550/arxiv.2605.14164","title":"Unsteady Metrics and Benchmarking Cultures of AI Model Builders","display_name":"Unsteady Metrics and Benchmarking Cultures of AI Model Builders","publication_year":2026,"publication_date":"2026-05-13","ids":{"openalex":"https://openalex.org/W7161287961","doi":"https://doi.org/10.48550/arxiv.2605.14164"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.14164","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14164","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.14164","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014487236","display_name":"Stefan Baack","orcid":"https://orcid.org/0000-0002-2464-7699"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Baack, Stefan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5117851054","display_name":"Christo Buschek","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Buschek, Christo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134932095","display_name":"Maty Bohacek","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bohacek, Maty","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.3138999938964844,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.3138999938964844,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.18449999392032623,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.08780000358819962,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8438000082969666},{"id":"https://openalex.org/keywords/terminology","display_name":"Terminology","score":0.692300021648407},{"id":"https://openalex.org/keywords/framing","display_name":"Framing (construction)","score":0.5885999798774719},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5066999793052673},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.45730000734329224},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.43389999866485596}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8438000082969666},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7035999894142151},{"id":"https://openalex.org/C547195049","wikidata":"https://www.wikidata.org/wiki/Q1725664","display_name":"Terminology","level":2,"score":0.692300021648407},{"id":"https://openalex.org/C169087156","wikidata":"https://www.wikidata.org/wiki/Q2131593","display_name":"Framing (construction)","level":2,"score":0.5885999798774719},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5264000296592712},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5066999793052673},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4625000059604645},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.45730000734329224},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.43389999866485596},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4104999899864197},{"id":"https://openalex.org/C199033989","wikidata":"https://www.wikidata.org/wiki/Q1318295","display_name":"Narrative","level":2,"score":0.38659998774528503},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.38370001316070557},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.33340001106262207},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2922999858856201},{"id":"https://openalex.org/C171686336","wikidata":"https://www.wikidata.org/wiki/Q3532085","display_name":"Topic model","level":2,"score":0.2822999954223633},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.260699987411499},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2572000026702881}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.14164","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14164","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.14164","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14164","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"primary":[1],"way":[2],"to":[3,19,57,96,164,177,212],"establish":[4],"and":[5,10,22,46,59,72,121,236],"compare":[6],"competencies":[7,143],"in":[8,85,124],"foundation":[9],"generative":[11],"AI":[12,90],"models":[13],"has":[14],"shifted":[15],"from":[16,87],"peer-reviewed":[17],"literature":[18],"press":[20],"releases":[21,84],"company":[23],"blog":[24],"posts,":[25],"where":[26],"model":[27,54,83],"builders":[28,55],"highlight":[29],"results":[30,202],"on":[31,148,172],"selected":[32],"benchmarks.":[33],"These":[34],"artifacts":[35],"now":[36],"largely":[37],"define":[38],"the":[39,42,47,98,183],"state":[40],"of":[41,77,112,168,205],"art":[43],"for":[44],"researchers":[45],"public.":[48],"Despite":[49],"their":[50,149],"prominence,":[51],"which":[52],"benchmarks":[53,79,114,139,196,229],"choose":[56],"highlight,":[58],"what":[60,173],"they":[61],"communicate":[62],"through":[63],"this":[64],"selection,":[65],"is":[66,182],"underexamined.":[67],"To":[68,151],"investigate,":[69],"we":[70,156],"introduce":[71],"open-source":[73],"Benchmarking-Cultures-25,":[74],"a":[75,103,118,158,165],"dataset":[76],"231":[78],"highlighted":[80,113,228],"across":[81],"139":[82],"2025":[86],"11":[88],"major":[89],"builders,":[91,146],"alongside":[92],"an":[93],"interactive":[94],"tool":[95],"explore":[97],"data.":[99],"Our":[100],"analysis":[101,192],"reveals":[102],"fragmented":[104],"evaluation":[105],"landscape":[106],"with":[107],"limited":[108],"cross-model":[109],"comparability:":[110],"63.2%":[111],"are":[115,140],"used":[116],"by":[117,144],"single":[119],"builder,":[120],"38.5%":[122],"appear":[123],"just":[125],"one":[126],"release.":[127],"Few":[128],"achieve":[129],"widespread":[130],"use":[131],"(e.g.,":[132],"GPQA":[133],"Diamond,":[134],"LiveCodeBench,":[135],"AIME":[136],"2025).":[137],"Moreover,":[138],"attributed":[141],"different":[142,145],"depending":[147],"narrative.":[150],"disentangle":[152],"these":[153],"conflicting":[154],"presentations,":[155],"develop":[157],"unified":[159],"taxonomy":[160],"mapping":[161],"diverging":[162],"terminology":[163],"shared":[166],"framework":[167],"measured":[169],"signals":[170],"based":[171],"benchmark":[174],"authors":[175,210],"claim":[176,211],"measure.":[178],"\"General":[179],"knowledge":[180,214],"application\"":[181],"second":[184],"most":[185],"popular,":[186],"yet":[187,218],"vaguely":[188],"defined,":[189],"category.":[190],"Qualitative":[191],"shows":[193],"many":[194],"such":[195],"deemphasize":[197],"construct":[198],"validity,":[199],"instead":[200],"framing":[201],"as":[203,232,238],"indicators":[204],"progress":[206],"toward":[207],"AGI.":[208],"Their":[209],"measure":[213],"or":[215],"reasoning":[216],"broadly,":[217],"mostly":[219],"evaluate":[220],"STEM":[221],"subjects":[222],"(especially":[223],"math).":[224],"We":[225],"argue":[226],"that":[227],"function":[230],"less":[231],"standardized":[233],"measurement":[234],"tools":[235],"more":[237],"flexible":[239],"narrative":[240],"devices":[241],"prioritizing":[242],"market":[243],"positioning":[244],"over":[245],"scientific":[246],"evaluation.":[247],"Data:":[248],"https://hf.co/datasets/matybohacek/benchmarking-cultures-25;":[249],"tool:":[250],"https://bench-cultures.net.":[251]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-16T00:00:00"}