{"id":"https://openalex.org/W7165002633","doi":"https://doi.org/10.48550/arxiv.2606.17930","title":"How Inference Compute Shapes Frontier LLM Evaluation","display_name":"How Inference Compute Shapes Frontier LLM Evaluation","publication_year":2026,"publication_date":"2026-06-16","ids":{"openalex":"https://openalex.org/W7165002633","doi":"https://doi.org/10.48550/arxiv.2606.17930"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.17930","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.17930","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.17930","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138801363","display_name":"Jessica McFadyen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"McFadyen, Jessica","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138778279","display_name":"Ole Jorgensen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jorgensen, Ole","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087769785","display_name":"Harry Coppock","orcid":"https://orcid.org/0000-0003-2404-8319"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Coppock, Harry","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002628114","display_name":"Ke Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Kevin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138829712","display_name":"Cozmin Ududec","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ududec, Cozmin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.310699999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.310699999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.1987999975681305,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.08049999922513962,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.6970999836921692},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.641700029373169},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6028000116348267},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5792999863624573},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.5788999795913696},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.5619999766349792},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.45329999923706055},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.43479999899864197}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7682999968528748},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6970999836921692},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.641700029373169},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6028000116348267},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5792999863624573},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5788999795913696},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.5619999766349792},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.45329999923706055},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.43479999899864197},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4198000133037567},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39579999446868896},{"id":"https://openalex.org/C51485801","wikidata":"https://www.wikidata.org/wiki/Q16966861","display_name":"Efficient frontier","level":3,"score":0.382999986410141},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.3409000039100647},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.33739998936653137},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.33719998598098755},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.32519999146461487},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3124000132083893},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.3109999895095825},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.25690001249313354},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.25429999828338623},{"id":"https://openalex.org/C2778915421","wikidata":"https://www.wikidata.org/wiki/Q3643177","display_name":"Performance improvement","level":2,"score":0.2531000077724457}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.17930","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.17930","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.17930","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.17930","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.4626989960670471}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"AI":[0],"evaluations":[1,41,144,213],"are":[2,207],"shifting":[3],"toward":[4],"harder":[5,164],"tasks":[6,165],"that":[7,51,204,212],"benefit":[8],"from":[9],"longer":[10],"trajectories":[11],"involving":[12],"tool":[13],"use":[14,88],"and":[15,29,85,102,140,166,194,227],"iterative":[16],"problem":[17],"solving.":[18],"As":[19],"a":[20,46,89,218,232],"result,":[21],"performance":[22,44,128,157],"is":[23],"increasingly":[24,146],"sensitive":[25],"to":[26,71],"the":[27,56,61,109,186],"amount":[28],"allocation":[30],"of":[31,188,220],"compute":[32,235],"available":[33],"at":[34,45,158,237],"test":[35,66],"time":[36],"(\"inference":[37],"compute\").":[38],"Yet":[39],"many":[40],"still":[42],"report":[43,215],"single":[47],"restrictive":[48],"budget,":[49],"meaning":[50],"low":[52],"scores":[53,206],"may":[54],"reflect":[55],"evaluation":[57],"setup":[58,91],"rather":[59],"than":[60],"model's":[62],"underlying":[63],"capability.":[64],"To":[65],"this,":[67],"we":[68],"evaluate":[69],"up":[70],"12":[72],"frontier":[73,148],"language":[74],"models":[75,151,154],"on":[76,129],"seven":[77],"challenging":[78],"benchmarks":[79,130,172],"spanning":[80],"software":[81],"engineering,":[82],"mathematics,":[83],"medicine,":[84],"cybersecurity.":[86],"We":[87,117,209],"controlled":[90],"combining":[92],"three":[93,119],"simple":[94],"inference-scaling":[95,176],"interventions:":[96],"larger":[97,123,189],"token":[98,124,190],"budgets,":[99,160,191,239],"context":[100],"compaction,":[101],"repeated":[103,180],"submission":[104,181],"attempts,":[105],"guided":[106],"either":[107],"by":[108,113,198],"model":[110,229],"itself":[111],"or":[112,243],"minimal":[114],"correctness":[115],"feedback.":[116],"find":[118],"main":[120],"results.":[121],"First,":[122],"budgets":[125],"substantially":[126],"improve":[127],"across":[131],"multiple":[132],"domains,":[133],"including":[134],"cybersecurity,":[135],"FrontierMath,":[136],"Humanity's":[137],"Last":[138],"Exam,":[139],"TerminalBench.":[141],"Second,":[142],"fixed-budget":[143],"can":[145],"understate":[147],"capability":[149,216],"as":[150,217],"advance.":[152],"Newer":[153],"reach":[155],"higher":[156],"large":[159,233],"where":[161],"they":[162],"unlock":[163],"solve":[167],"them":[168],"more":[169],"reliably.":[170],"Third,":[171],"differ":[173],"in":[174,241],"which":[175],"methods":[177],"help":[178],"most:":[179],"broadly":[182],"improves":[183],"performance,":[184],"but":[185],"value":[187],"external":[192],"feedback,":[193],"parallel":[195],"attempts":[196],"varies":[197],"benchmark.":[199],"Overall,":[200],"our":[201],"results":[202],"show":[203],"benchmark":[205],"protocol-dependent.":[208],"therefore":[210],"argue":[211],"should":[214],"function":[219],"inference-time":[221],"compute,":[222],"specify":[223],"protocol":[224],"choices":[225],"explicitly,":[226],"compare":[228],"generations":[230],"over":[231],"shared":[234],"range":[236],"matched":[238],"especially":[240],"safety-":[242],"policy-relevant":[244],"settings.":[245]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-18T00:00:00"}