{"id":"https://openalex.org/W7214714701","doi":"https://doi.org/10.48550/arxiv.2609.34683","title":"AgentPerfBench: A Benchmarking and Evaluation Suite for Inference Performance of Agentic LLMs","display_name":"AgentPerfBench: A Benchmarking and Evaluation Suite for Inference Performance of Agentic LLMs","publication_year":2026,"publication_date":"2026-09-28","ids":{"openalex":"https://openalex.org/W7214714701","doi":"https://doi.org/10.48550/arxiv.2609.34683"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.34683","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.34683","pdf_url":"https://arxiv.org/pdf/2609.34683","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.34683","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5147443732","display_name":"Cheuk Hang Lau","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Lau, Cheuk Hang","raw_affiliation_strings":["Imperial College London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Imperial College London","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153475864","display_name":"Zeyu Cao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Cao, Zeyu","raw_affiliation_strings":["University of Cambridge"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153416824","display_name":"Kevin Wong Cheuk Yin","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Kevin Wong Cheuk","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153457903","display_name":"Yao Lai","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Lai, Yao","raw_affiliation_strings":["University of Cambridge"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153447819","display_name":"Haoran Wu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Wu, Haoran","raw_affiliation_strings":["University of Cambridge"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045638679","display_name":"Nicholas D. Lane","orcid":"https://orcid.org/0000-0002-2728-8273","observed_orcids":["https://orcid.org/0000-0002-2728-8273"]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Lane, Nicholas D.","raw_affiliation_strings":["University of Cambridge"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040762162","display_name":"Robert Mullins","orcid":"https://orcid.org/0000-0002-8393-2748","observed_orcids":["https://orcid.org/0000-0002-8393-2748"]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mullins, Robert D.","raw_affiliation_strings":["University of Cambridge"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069844959","display_name":"Ilia Shumailov","orcid":"https://orcid.org/0000-0003-3100-0727","observed_orcids":["https://orcid.org/0000-0003-3100-0727"]},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Shumailov, Ilia","raw_affiliation_strings":["University of Oxford"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oxford","institution_ids":["https://openalex.org/I40120149"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5153475538","display_name":"Yiren Zhao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Zhao, Yiren","raw_affiliation_strings":["Imperial College London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Imperial College London","institution_ids":["https://openalex.org/I47508984"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9653000235557556,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9653000235557556,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.00800000037997961,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.00419999985024333,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/agentic-llms","display_name":"agentic LLMs","score":0.8240000009536743},{"id":"https://openalex.org/keywords/tool-calling","display_name":"tool calling","score":0.8640000224113464},{"id":"https://openalex.org/keywords/context-length","display_name":"context length","score":0.6549999713897705},{"id":"https://openalex.org/keywords/synthetic-workload-generation","display_name":"synthetic workload generation","score":0.6299999952316284},{"id":"https://openalex.org/keywords/hardware-performance-evaluation","display_name":"hardware performance evaluation","score":0.3869999945163727},{"id":"https://openalex.org/keywords/roofline-model","display_name":"roofline model","score":0.8759999871253967},{"id":"https://openalex.org/keywords/memory-bandwidth","display_name":"memory bandwidth","score":0.6110000014305115},{"id":"https://openalex.org/keywords/swe-bench","display_name":"SWE-bench","score":0.7599999904632568}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8468000292778015},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7674999833106995},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.7587000131607056},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.593999981880188},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5698000192642212},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5135999917984009},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.44279998540878296},{"id":"https://openalex.org/C2780575108","wikidata":"https://www.wikidata.org/wiki/Q7316652","display_name":"Retargeting","level":2,"score":0.41589999198913574},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4138000011444092},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3723999857902527},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.3425000011920929},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.3273000121116638},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.32109999656677246},{"id":"https://openalex.org/C2908650547","wikidata":"https://www.wikidata.org/wiki/Q20999234","display_name":"Intrinsics","level":2,"score":0.311599999666214},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3102000057697296},{"id":"https://openalex.org/C188045654","wikidata":"https://www.wikidata.org/wiki/Q17148339","display_name":"Memory bandwidth","level":2,"score":0.3021000027656555},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.2962999939918518},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2815999984741211},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.28029999136924744},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.27559998631477356},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.2720000147819519},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.26089999079704285},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.2549999952316284}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.34683","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.34683","pdf_url":"https://arxiv.org/pdf/2609.34683","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.34683","doi":"https://doi.org/10.48550/arxiv.2609.34683","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.34683","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.34683","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.34683","pdf_url":"https://arxiv.org/pdf/2609.34683","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214714701.pdf","grobid_xml":"https://content.openalex.org/works/W7214714701.grobid-xml"},"referenced_works_count":3,"referenced_works":["https://openalex.org/W4415797693","https://openalex.org/W7125193990","https://openalex.org/W7147097955"],"related_works":["https://openalex.org/W7204135284","https://openalex.org/W7170286594","https://openalex.org/W7203728158","https://openalex.org/W7119224602","https://openalex.org/W7213529265","https://openalex.org/W7204604445","https://openalex.org/W7125191802","https://openalex.org/W7220969866","https://openalex.org/W7162694930","https://openalex.org/W4417534084"],"abstract_inverted_index":{"The":[0,213],"optimization":[1],"of":[2,95,115],"LLM":[3,47],"serving":[4],"engines,":[5],"such":[6,83],"as":[7,84],"vLLM":[8],"and":[9,18,57,86,105,120,134,169,186,209,245],"SGLang,":[10],"is":[11],"largely":[12],"benchmark-driven:":[13],"optimizations,":[14],"scheduling":[15],"policies,":[16],"hardware":[17,154,178,227],"system":[19],"designs":[20],"are":[21,49],"all":[22],"selected":[23],"based":[24],"on":[25,42,97,137,225],"representative":[26,129],"workloads.":[27,46],"However,":[28],"a":[29,70,196],"significant":[30],"mismatch":[31],"has":[32],"emerged":[33],"in":[34,184,205,241],"the":[35,125,238],"agentic":[36,74,81,232],"era.":[37],"Existing":[38],"benchmarks":[39,148,244],"primarily":[40],"focus":[41],"simple":[43],"single-turn":[44],"chatbot":[45],"applications":[48],"increasingly":[50],"agentic:":[51],"coding":[52],"agents,":[53],"terminal":[54],"execution":[55],"systems,":[56],"tool-use":[58],"agents":[59],"issue":[60],"multi-turn":[61,98],"requests":[62],"with":[63,230],"growing":[64],"context":[65,107],"lengths.":[66,108],"We":[67,180],"introduce":[68],"AgentPerfBench,":[69],"benchmark":[71],"suite":[72,215],"for":[73,132,156,165],"inference.":[75],"It":[76],"uses":[77],"real":[78,126,153],"traces":[79,193],"from":[80,112,124],"benchmarks,":[82],"SWE-Bench":[85],"TerminalBench,":[87],"alongside":[88],"standard":[89],"chat":[90],"baselines.":[91],"This":[92],"enables":[93],"benchmarking":[94,214],"models":[96],"tasks":[99],"involving":[100],"tool":[101],"calling,":[102],"skill":[103],"utilization,":[104],"increasing":[106],"AgentPerfBench":[109],"also":[110],"samples":[111],"empirical":[113],"distributions":[114],"input":[116],"length,":[117,119],"output":[118],"turn":[121],"count":[122],"derived":[123],"traces,":[127],"generating":[128],"synthetic":[130],"profiles":[131],"cheap":[133],"accurate":[135],"measurements":[136],"new":[138,197],"hardware.":[139],"In":[140],"addition,":[141],"we":[142],"further":[143],"find":[144],"that":[145,201],"several":[146],"existing":[147],"fail":[149],"to":[150,194,220],"accurately":[151],"reflect":[152],"performance":[155,174],"two":[157],"key":[158],"reasons:":[159],"1)":[160],"they":[161,171],"do":[162],"not":[163],"account":[164],"realistic":[166],"context-length":[167],"growth,":[168],"2)":[170],"measure":[172],"inference":[173,243],"without":[175],"operating":[176],"at":[177],"saturation.":[179],"discuss":[181],"these":[182,235],"issues":[183],"detail":[185],"provide":[187],"rich":[188],"kernel-level":[189],"Nsight":[190],"Compute":[191],"(NCU)":[192],"construct":[195],"multi-dimensional":[198],"roofline":[199,252],"model":[200],"captures":[202],"hardware-system":[203],"limitations":[204],"both":[206],"memory":[207,210],"bandwidth":[208],"capacity":[211],"footprint.":[212],"then":[216],"includes":[217],"automated":[218],"scripts":[219],"identify":[221],"potential":[222],"bottleneck":[223],"conditions":[224],"emerging":[226],"when":[228],"evaluated":[229],"diverse":[231],"traces.":[233],"Together,":[234],"contributions":[236],"quantify":[237],"chat-to-agentic":[239],"gap":[240],"current":[242],"characterise":[246],"per-kernel":[247],"GPU":[248],"resource":[249],"utilisation":[250],"via":[251],"analysis.":[253]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-30T00:00:00"}