{"id":"https://openalex.org/W7167204904","doi":"https://doi.org/10.48550/arxiv.2607.01254","title":"The Benchmark Ceiling: Human Judgment, Evaluation Scarcity, and the Political Economy of AI Capability Measurement","display_name":"The Benchmark Ceiling: Human Judgment, Evaluation Scarcity, and the Political Economy of AI Capability Measurement","publication_year":2026,"publication_date":"2026-06-04","ids":{"openalex":"https://openalex.org/W7167204904","doi":"https://doi.org/10.48550/arxiv.2607.01254"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.01254","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.01254","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.01254","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140015775","display_name":"Mark Esposito","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Esposito, Mark","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5127970243","display_name":"Liu Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.8615000247955322,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.8615000247955322,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.022099999710917473,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14350","display_name":"Innovation, Sustainability, Human-Machine Systems","score":0.008700000122189522,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8363999724388123},{"id":"https://openalex.org/keywords/argument","display_name":"Argument (complex analysis)","score":0.5239999890327454},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.46129998564720154},{"id":"https://openalex.org/keywords/pareto-principle","display_name":"Pareto principle","score":0.44609999656677246},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.42410001158714294},{"id":"https://openalex.org/keywords/corporate-governance","display_name":"Corporate governance","score":0.40779998898506165},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.3871999979019165}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8363999724388123},{"id":"https://openalex.org/C98184364","wikidata":"https://www.wikidata.org/wiki/Q1780131","display_name":"Argument (complex analysis)","level":2,"score":0.5239999890327454},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.516700029373169},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.5097000002861023},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.46129998564720154},{"id":"https://openalex.org/C137635306","wikidata":"https://www.wikidata.org/wiki/Q182667","display_name":"Pareto principle","level":2,"score":0.44609999656677246},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.42410001158714294},{"id":"https://openalex.org/C39389867","wikidata":"https://www.wikidata.org/wiki/Q380767","display_name":"Corporate governance","level":2,"score":0.40779998898506165},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3871999979019165},{"id":"https://openalex.org/C51485801","wikidata":"https://www.wikidata.org/wiki/Q16966861","display_name":"Efficient frontier","level":3,"score":0.36070001125335693},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.33880001306533813},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31850001215934753},{"id":"https://openalex.org/C109747225","wikidata":"https://www.wikidata.org/wiki/Q815758","display_name":"Scarcity","level":2,"score":0.3160000145435333},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.3125},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.30469998717308044},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.3041999936103821},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.303600013256073},{"id":"https://openalex.org/C119898033","wikidata":"https://www.wikidata.org/wiki/Q3433888","display_name":"Ensemble forecasting","level":2,"score":0.28540000319480896},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.26989999413490295}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.01254","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.01254","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.01254","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.01254","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.6375659108161926,"display_name":"Reduced inequalities"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Benchmarks":[0],"are":[1,138],"the":[2,18,28,65,80,84,93,99,112,181,202,220,232],"primary":[3],"instruments":[4],"through":[5],"which":[6],"AI":[7,22],"capability":[8,155],"is":[9,24,41],"measured,":[10],"compared,":[11],"and":[12,37,157,192,235],"governed.":[13],"This":[14],"paper":[15,119],"argues":[16],"that":[17,38,46,173,180,193],"validity":[19,199],"of":[20,27,30,87,96,107,115,132,141,184],"frontier":[21,154,190],"benchmarks":[23,165],"a":[25,104,129],"function":[26],"quality":[29,40],"human":[31],"judgment":[32,74],"embedded":[33],"in":[34,44,64,123,177,198],"their":[35,146],"construction,":[36],"this":[39,79,121],"structurally":[42],"scarce":[43],"ways":[45],"standard":[47],"scaling":[48],"narratives":[49],"obscure.":[50],"As":[51,153],"foundation":[52],"models":[53,91],"approach":[54],"ceiling":[55,82],"performance":[56],"on":[57,150,207],"existing":[58],"evaluation":[59,88,227],"suites,":[60],"discriminating":[61],"signal":[62,89,134,175],"concentrates":[63,176],"hardest":[66],"benchmark":[67,81,133,151,195],"items,":[68,179],"precisely":[69],"those":[70],"requiring":[71],"elite":[72],"expert":[73,109],"to":[75,201],"design.":[76],"We":[77],"term":[78],"problem:":[83],"progressive":[85],"exhaustion":[86],"as":[90,158,167],"saturate":[92],"easy":[94],"majority":[95],"items":[97,186],"while":[98],"difficult":[100],"tail,":[101],"authored":[102],"by":[103],"thin":[105],"stratum":[106],"highly":[108],"evaluators,":[110],"remains":[111],"only":[113],"source":[114],"genuine":[116],"discrimination.":[117],"The":[118,170],"develops":[120],"argument":[122],"three":[124],"steps.":[125],"First,":[126],"we":[127,218,230],"present":[128],"formal":[130],"model":[131,143,171],"depreciation.":[135],"Benchmark":[136],"scores":[137],"public":[139],"signals":[140],"latent":[142],"quality,":[144],"but":[145],"precision":[147],"depends":[148],"endogenously":[149],"validity.":[152],"rises":[156,187],"contamination":[159],"or":[160],"strategic":[161],"optimization":[162],"increases,":[163],"fixed":[164],"depreciate":[166],"measurement":[168],"instruments.":[169],"shows":[172],"valid":[174],"hard-tail":[178],"replacement":[182],"cost":[183],"such":[185],"convexly":[188],"with":[189,224],"capability,":[191],"private":[194],"producers":[196],"underinvest":[197],"relative":[200],"social":[203],"optimum.":[204],"Second,":[205],"drawing":[206],"platform":[208],"data":[209],"from":[210],"micro1":[211],"covering":[212],"over":[213],"one":[214],"thousand":[215],"credentialed":[216],"professionals,":[217],"document":[219],"scarcity":[221],"premium":[222],"associated":[223],"high-judgment,":[225],"low-codifiability":[226],"labor.":[228],"Third,":[229],"develop":[231],"political":[233],"economy":[234],"governance":[236],"implications.":[237]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-04T00:00:00"}