{"id":"https://openalex.org/W7221827294","doi":"https://doi.org/10.48550/arxiv.2610.18454","title":"HEPTABench: A framework for portable and controlled agentic evaluations in high energy physics","display_name":"HEPTABench: A framework for portable and controlled agentic evaluations in high energy physics","publication_year":2026,"publication_date":"2026-10-13","ids":{"openalex":"https://openalex.org/W7221827294","doi":"https://doi.org/10.48550/arxiv.2610.18454"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.18454","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.18454","pdf_url":"https://arxiv.org/pdf/2610.18454","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.18454","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087932035","display_name":"Tony Menzo","orcid":"https://orcid.org/0000-0002-2013-4570","observed_orcids":["https://orcid.org/0000-0002-2013-4570"]},"institutions":[{"id":"https://openalex.org/I1314696892","display_name":"Fermi National Accelerator Laboratory","ror":"https://ror.org/020hgte69","country_code":"US","type":"facility","lineage":["https://openalex.org/I1314696892","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210114836"]},{"id":"https://openalex.org/I17301866","display_name":"University of Alabama","ror":"https://ror.org/03xrrjk67","country_code":"US","type":"education","lineage":["https://openalex.org/I17301866"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Menzo, Tony","raw_affiliation_strings":["Department of Physics and Astronomy , University of Alabama , Tuscaloosa , AL 35487 , USA","Fermi National Accelerator Laboratory , Batavia , IL 60510 , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Physics and Astronomy , University of Alabama , Tuscaloosa , AL 35487 , USA","institution_ids":["https://openalex.org/I17301866"]},{"raw_affiliation_string":"Fermi National Accelerator Laboratory , Batavia , IL 60510 , USA","institution_ids":["https://openalex.org/I1314696892"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001566718","display_name":"Alexander Roman","orcid":"https://orcid.org/0000-0003-2719-221X","observed_orcids":["https://orcid.org/0000-0003-2719-221X"]},"institutions":[{"id":"https://openalex.org/I17301866","display_name":"University of Alabama","ror":"https://ror.org/03xrrjk67","country_code":"US","type":"education","lineage":["https://openalex.org/I17301866"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Roman, Alexander","raw_affiliation_strings":["Department of Physics and Astronomy , University of Alabama , Tuscaloosa , AL 35487 , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Physics and Astronomy , University of Alabama , Tuscaloosa , AL 35487 , USA","institution_ids":["https://openalex.org/I17301866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054287026","display_name":"George Fleming","orcid":"https://orcid.org/0000-0002-4987-7167","observed_orcids":["https://orcid.org/0000-0002-4987-7167"]},"institutions":[{"id":"https://openalex.org/I1314696892","display_name":"Fermi National Accelerator Laboratory","ror":"https://ror.org/020hgte69","country_code":"US","type":"facility","lineage":["https://openalex.org/I1314696892","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210114836"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fleming, George T.","raw_affiliation_strings":["Fermi National Accelerator Laboratory , Batavia , IL 60510 , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fermi National Accelerator Laboratory , Batavia , IL 60510 , USA","institution_ids":["https://openalex.org/I1314696892"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148482072","display_name":"Gleyzer, Sergei, V","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I17301866","display_name":"University of Alabama","ror":"https://ror.org/03xrrjk67","country_code":"US","type":"education","lineage":["https://openalex.org/I17301866"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gleyzer, Sergei","raw_affiliation_strings":["Department of Physics and Astronomy , University of Alabama , Tuscaloosa , AL 35487 , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Physics and Astronomy , University of Alabama , Tuscaloosa , AL 35487 , USA","institution_ids":["https://openalex.org/I17301866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5151749583","display_name":"Konstantin Matchev","orcid":"https://orcid.org/0000-0003-4182-9096","observed_orcids":["https://orcid.org/0000-0003-4182-9096"]},"institutions":[{"id":"https://openalex.org/I17301866","display_name":"University of Alabama","ror":"https://ror.org/03xrrjk67","country_code":"US","type":"education","lineage":["https://openalex.org/I17301866"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matchev, Konstantin","raw_affiliation_strings":["Department of Physics and Astronomy , University of Alabama , Tuscaloosa , AL 35487 , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Physics and Astronomy , University of Alabama , Tuscaloosa , AL 35487 , USA","institution_ids":["https://openalex.org/I17301866"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5151871346","display_name":"Stephen Mrenna","orcid":"https://orcid.org/0000-0001-8731-160X","observed_orcids":["https://orcid.org/0000-0001-8731-160X"]},"institutions":[{"id":"https://openalex.org/I1314696892","display_name":"Fermi National Accelerator Laboratory","ror":"https://ror.org/020hgte69","country_code":"US","type":"facility","lineage":["https://openalex.org/I1314696892","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210114836"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mrenna, Stephen","raw_affiliation_strings":["Fermi National Accelerator Laboratory , Batavia , IL 60510 , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fermi National Accelerator Laboratory , Batavia , IL 60510 , USA","institution_ids":["https://openalex.org/I1314696892"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.6195999979972839,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.6195999979972839,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.27469998598098755,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.016300000250339508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/agentic-systems","display_name":"agentic systems","score":0.8149999976158142},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6269999742507935},{"id":"https://openalex.org/keywords/particle-physics","display_name":"particle physics","score":0.6850000023841858},{"id":"https://openalex.org/keywords/benchmarking-framework","display_name":"benchmarking framework","score":0.8930000066757202},{"id":"https://openalex.org/keywords/ai-coding-agents","display_name":"coding agents","score":0.703000009059906},{"id":"https://openalex.org/keywords/two-body-decays","display_name":"two-body decays","score":0.7960000038146973},{"id":"https://openalex.org/keywords/symbolic-reasoning","display_name":"symbolic reasoning","score":0.5320000052452087},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.7820000052452087},{"id":"https://openalex.org/keywords/agent-trajectories","display_name":"agent trajectories","score":0.47699999809265137},{"id":"https://openalex.org/keywords/tool-interfaces","display_name":"tool interfaces","score":0.3490000069141388},{"id":"https://openalex.org/keywords/claude-haiku","display_name":"Claude Haiku","score":0.7319999933242798},{"id":"https://openalex.org/keywords/gpt-5-5","display_name":"GPT-5.5","score":0.8769999742507935},{"id":"https://openalex.org/keywords/numerical-accuracy","display_name":"numerical accuracy","score":0.35899999737739563}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7652999758720398},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6560999751091003},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5562999844551086},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5070000290870667},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.47029998898506165},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.3855000138282776},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35019999742507935},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.33719998598098755},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.3296999931335449},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.3271999955177307},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3163999915122986},{"id":"https://openalex.org/C186370098","wikidata":"https://www.wikidata.org/wiki/Q442787","display_name":"Energy (signal processing)","level":2,"score":0.3093000054359436},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.30149999260902405},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.29789999127388},{"id":"https://openalex.org/C120524526","wikidata":"https://www.wikidata.org/wiki/Q1709148","display_name":"Reboot","level":2,"score":0.2858999967575073},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.2831000089645386},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.2799000144004822},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2732999920845032},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.2572999894618988},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2526000142097473},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.2508000135421753}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.18454","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.18454","pdf_url":"https://arxiv.org/pdf/2610.18454","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.18454","doi":"https://doi.org/10.48550/arxiv.2610.18454","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.18454","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.18454","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.18454","pdf_url":"https://arxiv.org/pdf/2610.18454","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2011917337","display_name":null,"funder_award_id":"89243024CSC000002","funder_id":"https://openalex.org/F4320338288","funder_display_name":"Fermilab"},{"id":"https://openalex.org/G2376186588","display_name":null,"funder_award_id":"89243024CSC000002","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G47680417","display_name":null,"funder_award_id":"DE-SC0026347","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G6471802762","display_name":null,"funder_award_id":"89243024CSC000002","funder_id":"https://openalex.org/F4320337511","funder_display_name":"High Energy Physics"}],"funders":[{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320332359","display_name":"Office of Science","ror":"https://ror.org/00mmn6b08"},{"id":"https://openalex.org/F4320337511","display_name":"High Energy Physics","ror":"https://ror.org/035m6g344"},{"id":"https://openalex.org/F4320338288","display_name":"Fermilab","ror":"https://ror.org/020hgte69"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7221827294.pdf","grobid_xml":"https://content.openalex.org/works/W7221827294.grobid-xml"},"referenced_works_count":1,"referenced_works":["https://openalex.org/W4292545695"],"related_works":["https://openalex.org/W7161237792","https://openalex.org/W7206185287","https://openalex.org/W7214775463","https://openalex.org/W7204748981","https://openalex.org/W7118539294","https://openalex.org/W7166657907","https://openalex.org/W7163131925","https://openalex.org/W7159956031","https://openalex.org/W7101416451","https://openalex.org/W7134993047"],"abstract_inverted_index":{"Quantifying":[0],"the":[1,38,88,104,130,175,206,212],"capability":[2],"and":[3,48,62,71,77,85,92,99,149,214],"reliability":[4,215],"of":[5,51,177,201,208,216],"agentic":[6,52,202],"systems":[7],"in":[8,106,191],"high-energy":[9],"physics":[10],"(HEP)":[11],"requires":[12,29],"repeated":[13],"evaluations":[14,50],"under":[15],"controlled":[16,49],"configurations.":[17],"Establishing":[18],"whether":[19],"changes":[20],"to":[21,32,186],"models,":[22,59],"tools,":[23],"or":[24],"harnesses":[25],"improve":[26,211],"performance":[27],"also":[28],"these":[30,142],"components":[31],"be":[33],"varied":[34],"independently.":[35],"We":[36,102,138],"present":[37],"High":[39],"Energy":[40],"Physics":[41],"Tool-and-Agent":[42],"Benchmarking":[43],"(HEPTABench)":[44],"framework":[45,66,89,105],"for":[46,153,180],"portable":[47],"systems,":[53,123,203],"with":[54,74,129,194],"separately":[55],"configurable":[56],"tasks,":[57,166],"scaffolding,":[58],"providers,":[60],"harnesses,":[61],"domain":[63,136,144,172],"capabilities.":[64],"The":[65,110],"combines":[67],"HEP":[68,135,143],"benchmark":[69,108],"tasks":[70],"scientific":[72,218],"rubrics":[73],"experiment":[75,95,100],"execution":[76],"versioned":[78],"tool":[79],"interfaces.":[80],"To":[81],"support":[82],"inspection,":[83],"reproducibility,":[84],"post-hoc":[86],"grading,":[87],"prioritizes":[90],"recording":[91],"reporting":[93],"complete":[94],"specifications,":[96],"agent":[97],"trajectories,":[98],"artifacts.":[101],"demonstrate":[103],"two":[107],"campaigns.":[109],"first":[111],"campaign":[112,160],"evaluates":[113],"a":[114,158],"forward":[115],"long-lived-particle":[116],"reach":[117],"calculation":[118],"across":[119],"four":[120],"frontier":[121],"coding-agent":[122],"comparing":[124],"their":[125,217],"built-in":[126,195],"coding":[127],"tools":[128,132,145,173,196],"same":[131],"supplemented":[133],"by":[134],"tools.":[137],"show":[139,168],"that":[140,169,210],"including":[141],"improves":[146],"numerical":[147],"accuracy":[148,213],"reduces":[150],"trial-to-trial":[151],"dispersion":[152],"every":[154,192],"tested":[155],"system.":[156],"In":[157],"complementary":[159],"covering":[161],"eight":[162],"symbolic":[163,171],"two-body":[164],"decay":[165],"we":[167],"adding":[170],"increases":[174],"fraction":[176],"successful":[178],"trials":[179],"Claude":[181],"Haiku":[182],"4.5":[183],"from":[184],"$8\\%$":[185],"$79\\%$,":[187],"while":[188],"GPT-5.5":[189],"succeeds":[190],"trial":[193],"alone.":[197],"Through":[198],"systematic":[199],"evaluation":[200],"HEPTABench":[204],"enables":[205],"identification":[207],"factors":[209],"work.":[219]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-15T00:00:00"}