{"id":"https://openalex.org/W7164533287","doi":"https://doi.org/10.48550/arxiv.2606.13608","title":"AgentBeats: Agentifying Agent Assessment for Openness, Standardization, and Reproducibility","display_name":"AgentBeats: Agentifying Agent Assessment for Openness, Standardization, and Reproducibility","publication_year":2026,"publication_date":"2026-06-11","ids":{"openalex":"https://openalex.org/W7164533287","doi":"https://doi.org/10.48550/arxiv.2606.13608"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.13608","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.13608","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.13608","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138539683","display_name":"Xiaoyuan Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xiaoyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082583725","display_name":"Jianhong Tu","orcid":"https://orcid.org/0009-0001-1554-1614"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tu, Jianhong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138528887","display_name":"Yuqi Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yuqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138542749","display_name":"Siyuan Xie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xie, Siyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138511634","display_name":"Sihan Ren","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ren, Sihan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138536907","display_name":"Tianneng Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Tianneng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114415504","display_name":"Gal Gantar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gantar, Gal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052719801","display_name":"Evan Sandoval","orcid":"https://orcid.org/0000-0003-4879-5968"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sandoval, Evan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033836068","display_name":"D Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Donghyun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114369236","display_name":"Daniel Miao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Miao, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138507598","display_name":"Peter J. Gilbert","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gilbert, Peter J.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077372325","display_name":"Nick Hynes","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hynes, Nick","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127702847","display_name":"Mauro Staver","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Staver, Mauro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138528774","display_name":"Warren He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Warren","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138548654","display_name":"David Marn","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Marn, David","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064576859","display_name":"Andrew Low","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Low, Andrew","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138519542","display_name":"Xi Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Xi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002225429","display_name":"Elron Bandel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bandel, Elron","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051536706","display_name":"Michal Shmueli-Scheuer","orcid":"https://orcid.org/0000-0002-6386-8726"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shmueli-Scheuer, Michal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010725488","display_name":"Somasekhar Reddy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Reddy, Siva","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028239106","display_name":"Alexandre Drouin","orcid":"https://orcid.org/0000-0001-7718-0319"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Drouin, Alexandre","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040248599","display_name":"Alexandre Lacoste","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lacoste, Alexandre","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109517432","display_name":"R Radha Krishnan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krishnan, Ramayya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074699094","display_name":"Elham Tabassi","orcid":"https://orcid.org/0000-0001-9536-2722"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tabassi, Elham","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138506266","display_name":"Yu Su","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Su, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120726138","display_name":"Victor Barres","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barres, Victor","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100329686","display_name":"Chenguang Wang","orcid":"https://orcid.org/0000-0001-6155-0530"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Chenguang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138515295","display_name":"Wenbo Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Wenbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138545508","display_name":"Dawn Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Dawn","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.49219998717308044,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.49219998717308044,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.040300000458955765,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.030700000002980232,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.7879999876022339},{"id":"https://openalex.org/keywords/fidelity","display_name":"Fidelity","score":0.5314000248908997},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.5135999917984009},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4564000070095062},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.3734999895095825},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.33399999141693115}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7879999876022339},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6675000190734863},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.5314000248908997},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.5135999917984009},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4564000070095062},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3734999895095825},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.33399999141693115},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3255999982357025},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2888000011444092},{"id":"https://openalex.org/C41550386","wikidata":"https://www.wikidata.org/wiki/Q529909","display_name":"Multi-agent system","level":2,"score":0.287200003862381},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.274399995803833},{"id":"https://openalex.org/C2780871342","wikidata":"https://www.wikidata.org/wiki/Q7883752","display_name":"Underpinning","level":2,"score":0.26579999923706055},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.25279998779296875}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.13608","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.13608","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.13608","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.13608","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Agent":[0,49],"systems":[1],"are":[2],"advancing":[3],"quickly":[4],"across":[5,30,163,178,233],"domains,":[6],"but":[7],"their":[8],"evaluation":[9,53,194],"remains":[10],"fragmented.":[11],"Most":[12],"benchmarks":[13],"rely":[14],"on":[15,139,188],"fixed,":[16],"LLM-centric":[17],"harnesses":[18],"that":[19,101,131,158,175,191,226],"require":[20],"heavy":[21],"integration,":[22],"create":[23],"test-production":[24],"mismatch,":[25],"and":[26,59,70,85,108,112,142,184,218,231,240,249],"limit":[27],"fair":[28],"comparison":[29],"diverse":[31],"agent":[32,106,211,251],"designs.":[33],"The":[34],"root":[35],"problem":[36],"is":[37,54],"the":[38,83,88,198],"lack":[39],"of":[40,123,182],"an":[41],"open,":[42,247],"agent-agnostic":[43],"assessment":[44,103,134],"interface.":[45],"We":[46,115],"advocate":[47],"Agentified":[48],"Assessment":[50],"(AAA),":[51],"where":[52],"performed":[55],"by":[56],"judge":[57,161],"agents":[58,162,170,190],"all":[60],"participants":[61],"interact":[62],"through":[63],"standardized":[64,133],"protocols:":[65],"A2A":[66],"for":[67,72,82,87],"task":[68],"management":[69],"MCP":[71],"tool":[73],"access.":[74],"Conventional":[75],"benchmarking":[76],"defines":[77],"two":[78,152],"separate":[79],"interfaces,":[80],"one":[81,86],"benchmark":[84],"agent,":[89],"while":[90,201],"AAA":[91,176,227,239],"only":[92],"needs":[93],"one;":[94],"this":[95],"yields":[96],"a":[97,120,154,179,185,214,219,243],"generic,":[98],"unified":[99],"framework":[100],"separates":[102],"logic":[104],"from":[105,171],"implementation":[107],"enables":[109],"reproducible,":[110],"interoperable,":[111],"multi-agent":[113],"evaluation.":[114],"further":[116],"introduce":[117],"AgentBeats":[118,241],"as":[119],"concrete":[121],"realization":[122],"AAA:":[124],"we":[125,150,224],"identify":[126],"five":[127],"practical":[128],"operation":[129],"modes":[130],"make":[132],"compatible":[135],"with":[136,167,197],"real-world":[137],"constraints":[138],"openness,":[140],"privacy,":[141],"reproducibility.":[143],"To":[144],"evaluate":[145],"our":[146],"design":[147],"at":[148,236],"scale,":[149],"conduct":[151],"studies:":[153],"five-month":[155],"open":[156],"competition":[157],"drew":[159],"298":[160],"12":[164],"categories":[165],"together":[166],"467":[168],"subject":[169],"independent":[172],"participants,":[173],"showing":[174],"applies":[177],"heterogeneous":[180,234],"range":[181],"benchmarks;":[183],"case":[186,222],"study":[187,217],"coding":[189,221],"confirms":[192],"agentified":[193],"preserves":[195],"fidelity":[196,232],"public":[199],"record":[200],"surfacing":[202],"previously":[203],"missing":[204],"head-to-head":[205],"results,":[206],"yielding":[207],"research":[208],"insights":[209],"about":[210],"design.":[212],"Combining":[213],"community-scale":[215],"field":[216],"controlled":[220],"study,":[223],"verify":[225],"delivers":[228],"coverage,":[229],"practicality,":[230],"scenarios":[235],"scale.":[237],"Together,":[238],"offer":[242],"clear":[244],"path":[245],"toward":[246],"standardized,":[248],"reproducible":[250],"assessment.":[252]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-13T00:00:00"}