{"id":"https://openalex.org/W7140322352","doi":"https://doi.org/10.48550/arxiv.2603.23160","title":"UniDial-EvalKit: A Unified Toolkit for Evaluating Multi-Faceted Conversational Abilities","display_name":"UniDial-EvalKit: A Unified Toolkit for Evaluating Multi-Faceted Conversational Abilities","publication_year":2026,"publication_date":"2026-03-24","ids":{"openalex":"https://openalex.org/W7140322352","doi":"https://doi.org/10.48550/arxiv.2603.23160"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2603.23160","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2603.23160","pdf_url":"https://arxiv.org/pdf/2603.23160","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2603.23160","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101812176","display_name":"Qi Jia","orcid":"https://orcid.org/0000-0001-5383-0065","observed_orcids":["https://orcid.org/0000-0001-5383-0065"]},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jia, Qi","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025780527","display_name":"Haodong Zhao","orcid":"https://orcid.org/0000-0002-4405-1649","observed_orcids":["https://orcid.org/0000-0002-4405-1649"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhao, Haodong","raw_affiliation_strings":["Shanghai Jiao Tong University \u22c6 https ://github.com/ UniDial / UniDial-EvalKit"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University \u22c6 https ://github.com/ UniDial / UniDial-EvalKit","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122387157","display_name":"Dun Pei","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pei, Dun","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130572098","display_name":"Xiujie Song","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Song, Xiujie","raw_affiliation_strings":["Shanghai Jiao Tong University \u22c6 https ://github.com/ UniDial / UniDial-EvalKit"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University \u22c6 https ://github.com/ UniDial / UniDial-EvalKit","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058270058","display_name":"Ye Shen","orcid":"https://orcid.org/0000-0003-0782-921X","observed_orcids":["https://orcid.org/0000-0003-0782-921X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Ye","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130543216","display_name":"Shibo Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang, Shibo","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130562372","display_name":"Zijian Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen, Zijian","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory","Shanghai Jiao Tong University \u22c6 https ://github.com/ UniDial / UniDial-EvalKit"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4391012619"]},{"raw_affiliation_string":"Shanghai Jiao Tong University \u22c6 https ://github.com/ UniDial / UniDial-EvalKit","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5146560235","display_name":"Zicheng Zhang","orcid":"https://orcid.org/0000-0002-7247-7938","observed_orcids":["https://orcid.org/0000-0002-7247-7938"]},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhang, Zicheng","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5159907576","display_name":"Xiangyang Zhu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhu, Xiangyang","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064168853","display_name":"Guangtao Zhai","orcid":"https://orcid.org/0000-0001-8165-9322","observed_orcids":["https://orcid.org/0000-0001-8165-9322"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhai, Guangtao","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory","Shanghai Jiao Tong University \u22c6 https ://github.com/ UniDial / UniDial-EvalKit"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4391012619"]},{"raw_affiliation_string":"Shanghai Jiao Tong University \u22c6 https ://github.com/ UniDial / UniDial-EvalKit","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9829000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9829000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0035000001080334187,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0032999999821186066,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7919999957084656},{"id":"https://openalex.org/keywords/ai-agents","display_name":"AI agents","score":0.6079999804496765},{"id":"https://openalex.org/keywords/assessment-toolkit","display_name":"evaluation toolkit","score":0.6240000128746033},{"id":"https://openalex.org/keywords/benchmark-standardization","display_name":"benchmark standardization","score":0.6539999842643738},{"id":"https://openalex.org/keywords/parallel-generation","display_name":"parallel generation","score":0.7110000252723694},{"id":"https://openalex.org/keywords/memory-agents","display_name":"memory agents","score":0.8130000233650208},{"id":"https://openalex.org/keywords/memory-architecture","display_name":"memory architecture","score":0.40700000524520874}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.826200008392334},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8070999979972839},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6704999804496765},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.66839998960495},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.5986999869346619},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.48539999127388},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4578000009059906},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.4196999967098236},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.375},{"id":"https://openalex.org/C2780366209","wikidata":"https://www.wikidata.org/wiki/Q5170200","display_name":"Core model","level":2,"score":0.3650999963283539},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.3434999883174896},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.3375000059604645},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3314000070095062},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33009999990463257},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.3149000108242035},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2903999984264374},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.29019999504089355},{"id":"https://openalex.org/C188087704","wikidata":"https://www.wikidata.org/wiki/Q369577","display_name":"Standardization","level":2,"score":0.28519999980926514},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.27459999918937683},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.273499995470047},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.26910001039505005},{"id":"https://openalex.org/C2984074130","wikidata":"https://www.wikidata.org/wiki/Q73539779","display_name":"R package","level":2,"score":0.25049999356269836}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2603.23160","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2603.23160","pdf_url":"https://arxiv.org/pdf/2603.23160","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2603.23160","doi":"https://doi.org/10.48550/arxiv.2603.23160","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.23160","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2603.23160","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2603.23160","pdf_url":"https://arxiv.org/pdf/2603.23160","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7140322352.pdf","grobid_xml":"https://content.openalex.org/works/W7140322352.grobid-xml"},"referenced_works_count":7,"referenced_works":["https://openalex.org/W4399251648","https://openalex.org/W4404088778","https://openalex.org/W4415971869","https://openalex.org/W4417006854","https://openalex.org/W7119535842","https://openalex.org/W7127380112","https://openalex.org/W7130402983"],"related_works":["https://openalex.org/W4405254441","https://openalex.org/W7162694930","https://openalex.org/W7168591537","https://openalex.org/W7159956031","https://openalex.org/W4416982487","https://openalex.org/W7130534570","https://openalex.org/W4412945638","https://openalex.org/W7119534494","https://openalex.org/W4415797149","https://openalex.org/W4406733842"],"abstract_inverted_index":{"Benchmarking":[0],"large":[1],"language":[2],"models":[3],"(LLMs)":[4],"and":[5,32,83,101,156],"agents":[6,140],"in":[7,27,62],"multi-turn":[8],"interactive":[9,53],"scenarios":[10],"is":[11],"essential":[12],"for":[13,51],"understanding":[14],"their":[15],"practical":[16],"capabilities.":[17],"However,":[18],"existing":[19],"evaluation":[20,33,49,77,97,118],"protocols":[21],"are":[22],"highly":[23],"heterogeneous,":[24],"differing":[25],"significantly":[26],"dataset":[28],"formats,":[29],"model":[30],"interfaces,":[31],"pipelines,":[34],"which":[35],"severely":[36],"impedes":[37],"systematic":[38],"comparison.":[39],"In":[40],"this":[41],"work,":[42],"we":[43,114],"present":[44],"UniDial-EvalKit":[45],"(UDE),":[46],"a":[47,72,80,88],"unified":[48],"toolkit":[50],"assessing":[52],"AI":[54],"systems.":[55],"The":[56],"core":[57],"contribution":[58],"of":[59],"UDE":[60],"lies":[61],"its":[63],"holistic":[64],"unification:":[65],"it":[66],"standardizes":[67],"heterogeneous":[68],"data":[69],"formats":[70],"into":[71],"universal":[73],"schema,":[74],"streamlines":[75],"complex":[76],"pipelines":[78],"through":[79,98],"modular":[81],"architecture,":[82],"aligns":[84],"metric":[85],"calculations":[86],"under":[87],"hierarchical":[89],"scoring":[90],"aggregation.":[91],"It":[92],"also":[93],"supports":[94],"efficient":[95],"large-scale":[96],"parallel":[99],"generation":[100],"scoring,":[102],"as":[103,105],"well":[104],"checkpoint":[106],"resume":[107],"to":[108,143],"eliminate":[109],"redundant":[110],"computation.":[111],"Leveraging":[112],"UDE,":[113],"conduct":[115],"an":[116],"extensive":[117],"across":[119,134],"diverse":[120],"multi-dimensional":[121],"benchmarks.":[122],"Our":[123],"empirical":[124],"analysis":[125],"shows":[126],"that":[127],"no":[128],"single":[129],"system":[130],"consistently":[131],"outperforms":[132],"others":[133],"all":[135],"benchmarks,":[136],"while":[137],"current":[138],"memory":[139,159],"often":[141],"fail":[142],"surpass":[144],"full-context":[145],"baselines.":[146],"Further":[147],"analyses":[148],"highlight":[149],"several":[150],"future":[151],"directions,":[152],"including":[153],"benchmark":[154],"deduplication":[155],"more":[157],"adaptive":[158],"architectures.":[160]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-03-26T00:00:00"}