{"id":"https://openalex.org/W4415797149","doi":"https://doi.org/10.52202/079017-2365","title":"AgentBoard: An Analytical Evaluation Board of Multi-turn LLM Agents","display_name":"AgentBoard: An Analytical Evaluation Board of Multi-turn LLM Agents","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4415797149","doi":"https://doi.org/10.52202/079017-2365"},"language":"en","primary_location":{"id":"doi:10.52202/079017-2365","doi":"https://doi.org/10.52202/079017-2365","is_oa":false,"landing_page_url":"https://doi.org/10.52202/079017-2365","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 37","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2401.13178","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Chang Ma","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chang Ma","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Junlei Zhang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Junlei Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103234468","display_name":"Zhihao Zhu","orcid":"https://orcid.org/0000-0001-5457-2352","observed_orcids":["https://orcid.org/0000-0001-5457-2352"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhihao Zhu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Cheng Yang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yujiu Yang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yujiu Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085787425","display_name":"Yaohui Jin","orcid":"https://orcid.org/0000-0001-6158-6277","observed_orcids":["https://orcid.org/0000-0001-6158-6277"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yaohui Jin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103239171","display_name":"Zhenzhong Lan","orcid":"https://orcid.org/0000-0003-4763-6148","observed_orcids":["https://orcid.org/0000-0003-4763-6148"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhenzhong Lan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Lingpeng Kong","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lingpeng Kong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5015879697","display_name":"Junxian He","orcid":"https://orcid.org/0009-0007-9559-6941","observed_orcids":["https://orcid.org/0009-0007-9559-6941"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Junxian He","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.2246,"has_fulltext":true,"cited_by_count":28,"citation_normalized_percentile":{"value":0.98431743,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"74325","last_page":"74362"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.21299999952316284,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.21299999952316284,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.15029999613761902,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.11659999936819077,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/agent-evaluation","display_name":"agent evaluation","score":0.5870000123977661},{"id":"https://openalex.org/keywords/evaluation-board","display_name":"evaluation board","score":0.6029999852180481},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8429999947547913}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4097000062465668},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.34540000557899475},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2703000009059906},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.25850000977516174},{"id":"https://openalex.org/C147494362","wikidata":"https://www.wikidata.org/wiki/Q2078905","display_name":"Troubleshooting","level":2,"score":0.25360000133514404},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.2393999993801117},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.2371000051498413},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.2361000031232834},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.23430000245571136},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.23330000042915344}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.52202/079017-2365","doi":"https://doi.org/10.52202/079017-2365","is_oa":false,"landing_page_url":"https://doi.org/10.52202/079017-2365","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 37","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2401.13178","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.13178","pdf_url":"https://arxiv.org/pdf/2401.13178","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-148441","doi":null,"is_oa":false,"landing_page_url":"http://www.scopus.com/record/display.url?eid=2-s2.0-105000484281&origin=inward","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"},{"id":"doi:10.48550/arxiv.2401.13178","doi":"https://doi.org/10.48550/arxiv.2401.13178","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2401.13178","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2401.13178","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.13178","pdf_url":"https://arxiv.org/pdf/2401.13178","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1445079709","display_name":null,"funder_award_id":"2021SHZDZX0102","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"}],"funders":[{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"},{"id":"https://openalex.org/F4320322999","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415797149.pdf","grobid_xml":"https://content.openalex.org/works/W4415797149.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7128518302","https://openalex.org/W7131893979","https://openalex.org/W4412877164","https://openalex.org/W4415289348","https://openalex.org/W7168591537","https://openalex.org/W4415995818","https://openalex.org/W7159956031","https://openalex.org/W7119224602","https://openalex.org/W7162219190","https://openalex.org/W4415068065"],"abstract_inverted_index":{"Evaluating":[0],"Large":[1],"Language":[2],"Models":[3],"(LLMs)":[4],"as":[5,115,117,158],"general-purpose":[6],"agents":[7,127,143],"is":[8,31],"essential":[9],"for":[10,128],"understanding":[11,76],"their":[12,16,150],"capabilities":[13,138],"and":[14,49,70,92,139,165],"facilitating":[15],"integration":[17],"into":[18],"practical":[19],"applications.":[20],"However,":[21],"the":[22,32,60,68,78,137,147,153,167],"evaluation":[23,55,95,100,120],"process":[24,69],"presents":[25],"substantial":[26],"challenges.":[27],"A":[28],"primary":[29],"obstacle":[30],"benchmarking":[33],"of":[34,77,101,126,141,149,169],"agent":[35,163],"performance":[36,151],"across":[37],"diverse":[38],"scenarios":[39],"within":[40],"a":[41,74,88,106,118,159],"unified":[42],"framework,":[43],"especially":[44],"in":[45],"maintaining":[46],"partially-observable":[47],"environments":[48],"ensuring":[50],"multi-round":[51],"interactions.":[52],"Moreover,":[53],"current":[54],"frameworks":[56],"mostly":[57],"focus":[58],"on":[59,136],"final":[61],"success":[62],"rate,":[63],"revealing":[64],"few":[65],"insights":[66],"during":[67],"failing":[71],"to":[72,98,152],"provide":[73],"deep":[75],"model":[79],"abilities.":[80],"To":[81],"address":[82],"these":[83],"challenges,":[84],"we":[85],"introduce":[86],"AgentBoard,":[87],"pioneering":[89],"comprehensive":[90,119],"benchmark":[91],"accompanied":[93],"open-source":[94],"framework":[96],"tailored":[97],"analytical":[99],"LLM":[102,142,171],"agents.":[103,172],"AgentBoard":[104,156],"offers":[105],"fine-grained":[107],"progress":[108],"rate":[109],"metric":[110],"that":[111,122],"captures":[112],"incremental":[113],"advancements":[114],"well":[116],"toolkit":[121],"features":[123],"easy":[124],"assessment":[125],"multi-faceted":[129],"analysis.":[130],"This":[131],"not":[132],"only":[133],"sheds":[134],"light":[135],"limitations":[140],"but":[144],"also":[145],"propels":[146],"interpretability":[148],"forefront.":[154],"Ultimately,":[155],"serves":[157],"step":[160],"towards":[161],"demystifying":[162],"behaviors":[164],"accelerating":[166],"development":[168],"stronger":[170]},"counts_by_year":[{"year":2026,"cited_by_count":18},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-02T09:45:03.842371","created_date":"2025-11-03T00:00:00"}