{"id":"https://openalex.org/W7165681683","doi":"https://doi.org/10.1145/3805689.3812240","title":"Unsteady Metrics and Benchmarking Cultures of AI Model Builders","display_name":"Unsteady Metrics and Benchmarking Cultures of AI Model Builders","publication_year":2026,"publication_date":"2026-06-23","ids":{"openalex":"https://openalex.org/W7165681683","doi":"https://doi.org/10.1145/3805689.3812240"},"language":null,"primary_location":{"id":"doi:10.1145/3805689.3812240","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805689.3812240","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805689.3812240","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014487236","display_name":"Stefan Baack","orcid":"https://orcid.org/0000-0002-2464-7699"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stefan Baack","raw_affiliation_strings":["Independent Researcher, Berlin, Germany"],"raw_orcid":"https://orcid.org/0000-0002-2464-7699","affiliations":[{"raw_affiliation_string":"Independent Researcher, Berlin, Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022473578","display_name":"Maty\u00e1\u0161 Boh\u00e1\u010dek","orcid":"https://orcid.org/0000-0001-8683-3692"},"institutions":[{"id":"https://openalex.org/I1743320","display_name":"Palo Alto University","ror":"https://ror.org/04f812k67","country_code":"US","type":"education","lineage":["https://openalex.org/I1743320"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Maty Bohacek","raw_affiliation_strings":["Stanford University, Palo Alto, USA"],"raw_orcid":"https://orcid.org/0000-0001-8683-3692","affiliations":[{"raw_affiliation_string":"Stanford University, Palo Alto, USA","institution_ids":["https://openalex.org/I1743320","https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5117851054","display_name":"Christo Buschek","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Christo Buschek","raw_affiliation_strings":["Independent Researcher, Vienna, Austria"],"raw_orcid":"https://orcid.org/0009-0001-4096-7465","affiliations":[{"raw_affiliation_string":"Independent Researcher, Vienna, Austria","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.94152074,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"840","last_page":"872"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.11649999767541885,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.11649999767541885,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.07800000160932541,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.053300000727176666,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6525999903678894},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.26669999957084656},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.2485000044107437},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.24320000410079956},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.24130000174045563}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6525999903678894},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5422999858856201},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.3763999938964844},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.335099995136261},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29919999837875366},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.26669999957084656},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2485000044107437},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.24320000410079956},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.24130000174045563},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.24079999327659607}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805689.3812240","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805689.3812240","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805689.3812240","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805689.3812240","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.531767725944519,"display_name":"Sustainable cities and communities","id":"https://metadata.un.org/sdg/11"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W2462906003","https://openalex.org/W2472097883","https://openalex.org/W3083410900","https://openalex.org/W3091454361","https://openalex.org/W3123282572","https://openalex.org/W3168194750","https://openalex.org/W3171654528","https://openalex.org/W4308885736","https://openalex.org/W4378189609","https://openalex.org/W4388481599","https://openalex.org/W4388926373","https://openalex.org/W4389156660","https://openalex.org/W4391136507","https://openalex.org/W4392822190","https://openalex.org/W4393216547","https://openalex.org/W4399404161","https://openalex.org/W4401043273","https://openalex.org/W4402671206","https://openalex.org/W4406840771","https://openalex.org/W4409348063","https://openalex.org/W4409415146","https://openalex.org/W4410034089","https://openalex.org/W4411119490","https://openalex.org/W4411661937","https://openalex.org/W4415797772","https://openalex.org/W4416036918"],"related_works":[],"abstract_inverted_index":{"The":[0],"primary":[1],"way":[2],"to":[3,21,67,110,122,191,204,279,325],"establish":[4],"and":[5,11,24,55,69,83,142,160,261,316,339],"compare":[6],"model":[7,29,64,94,102,140,148,171,306],"competencies":[8,168],"in":[9,96,145,220,254,262,285,305],"foundation":[10],"generative":[12],"AI":[13,101],"models":[14],"has":[15],"largely":[16,43],"moved":[17],"from":[18,98],"peer-reviewed":[19],"literature":[20,260],"press":[22],"releases":[23,95],"company":[25],"blog":[26],"posts,":[27],"where":[28],"builders":[30,65],"highlight":[31,68],"results":[32,239],"on":[33,174,199,297],"a":[34,86,123,138,184,192,327],"selection":[35,75],"of":[36,47,115,132,195,218,226,242,273,329],"benchmarks.":[37],"These":[38],"public-facing":[39],"industry":[40],"artifacts":[41,308],"now":[42],"define":[44],"the":[45,48,52,56,113,116,210,346],"state":[46],"art,":[49],"both":[50,253],"for":[51],"research":[53],"community":[54],"broader":[57],"public.":[58],"Despite":[59],"their":[60,175,238],"prominence,":[61],"which":[62],"benchmarks":[63,90,134,151,164,228,255,277,304],"choose":[66],"what":[70,200],"they":[71,236],"communicate":[72],"through":[73],"this":[74],"is":[76,209,251,342,349],"underexamined.":[77],"To":[78,177],"investigate":[79],"this,":[80],"we":[81,105,182,300],"introduce":[82],"open-source":[84],"Benchmarking-Cultures-25,":[85],"dataset":[87],"containing":[88],"231":[89],"highlighted":[91,133,303],"across":[92,291],"139":[93],"2025":[97],"11":[99],"major":[100],"builders.":[103],"Additionally,":[104],"publish":[106],"an":[107],"interactive":[108,347],"tool":[109,348],"visually":[111],"explore":[112],"relationships":[114],"collected":[117],"data.":[118],"Our":[119],"analysis":[120,225],"points":[121],"fragmented":[124],"evaluation":[125,338],"landscape":[126],"with":[127],"limited":[128],"cross-model":[129],"comparability:":[130],"63.2%":[131],"are":[135,165,323],"used":[136,324],"by":[137,169,266],"single":[139],"builder,":[141],"38.5%":[143],"appear":[144],"just":[146],"one":[147],"release.":[149],"Few":[150],"achieve":[152],"true":[153],"widespread":[154],"use":[155],"(e.g.,":[156],"GPQA":[157],"Diamond,":[158],"LiveCodeBench,":[159],"AIME":[161],"2025).":[162],"Moreover,":[163],"attributed":[166],"different":[167,170],"builders,":[172],"depending":[173],"narrative.":[176],"disentangle":[178],"these":[179,227,298],"conflicting":[180],"presentations,":[181],"develop":[183],"unified":[185],"taxonomy":[186],"that":[187,230,256,302,322,331],"maps":[188],"diverging":[189],"terminology":[190],"shared":[193],"framework":[194],"measured":[196],"signals":[197],"based":[198],"benchmark":[201,219],"authors":[202,272],"claim":[203,278],"measure.":[205],"\u201cGeneral":[206,274],"knowledge":[207,275,281],"application\u201d":[208,276],"second":[211],"most":[212],"popular,":[213],"yet":[214,287],"vaguely":[215],"defined,":[216],"category":[217],"our":[221],"dataset.":[222],"A":[223],"qualitative":[224],"revealed":[229],"many":[231],"deemphasize":[232],"construct":[233,326],"validity;":[234],"instead,":[235],"frame":[237],"as":[240,312,318],"indicators":[241],"progress":[243,330],"toward":[244],"Artificial":[245],"General":[246],"Intelligence":[247],"(AGI).":[248],"This":[249],"framing":[250],"evident":[252],"explicitly":[257],"cite":[258],"AGI":[259],"those":[263],"implicitly":[264],"shaped":[265],"its":[267],"surrounding":[268],"narratives.":[269],"In":[270],"addition,":[271],"measure":[280],"or":[282],"reasoning":[283],"capabilities":[284],"general,":[286],"mostly":[288],"evaluate":[289],"them":[290],"STEM":[292],"subjects":[293],"(especially":[294],"math).":[295],"Based":[296],"findings,":[299],"argue":[301],"release":[307],"currently":[309],"function":[310],"less":[311],"standardized":[313],"measurement":[314],"tools":[315],"more":[317],"flexible":[319],"narrative":[320],"devices":[321],"story":[328],"prioritizes":[332],"market":[333],"positioning":[334],"over":[335],"practical":[336],"scientific":[337],"comparison.":[340],"Data":[341],"available":[343,350],"at":[344,351],"https://hf.co/datasets/matybohacek/benchmarking-cultures-25;":[345],"https://bench-cultures.net.":[352]},"counts_by_year":[],"updated_date":"2026-06-25T06:16:34.145877","created_date":"2026-06-24T00:00:00"}