{"id":"https://openalex.org/W4415329203","doi":"https://doi.org/10.48550/arxiv.2505.15727","title":"VocalBench: Benchmarking the Vocal Conversational Abilities for Speech Interaction Models","display_name":"VocalBench: Benchmarking the Vocal Conversational Abilities for Speech Interaction Models","alternate_titles":[],"publication_year":2025,"publication_date":"2025-05-21","ids":{"openalex":"https://openalex.org/W4415329203","doi":"https://doi.org/10.48550/arxiv.2505.15727"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2505.15727","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.15727","pdf_url":"https://arxiv.org/pdf/2505.15727","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4415329203"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2505.15727","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051238464","display_name":"Heyang Liu","orcid":"https://orcid.org/0009-0007-0402-1078","observed_orcids":["https://orcid.org/0009-0007-0402-1078","https://orcid.org/0009-0007-1814-305X"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Liu, Heyang","raw_affiliation_strings":["Ant Group","Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group","institution_ids":["https://openalex.org/I4405270328"]},{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043861652","display_name":"Yuhao Wang","orcid":"https://orcid.org/0000-0002-6051-8659","observed_orcids":["https://orcid.org/0000-0002-6051-8659"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Wang, Yuhao","raw_affiliation_strings":["Ant Group","Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group","institution_ids":["https://openalex.org/I4405270328"]},{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017638299","display_name":"Ziyang Cheng","orcid":"https://orcid.org/0000-0002-4498-2443","observed_orcids":["https://orcid.org/0000-0002-4498-2443"]},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Cheng, Ziyang","raw_affiliation_strings":["Wuhan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Wuhan University","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Hongcheng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Liu, Hongcheng","raw_affiliation_strings":["Ant Group","Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group","institution_ids":["https://openalex.org/I4405270328"]},{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Yiqi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Li, Yiqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Hou, Yixuan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Hou, Yixuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081951212","display_name":"Ronghua Wu","orcid":"https://orcid.org/0000-0002-4454-2595","observed_orcids":["https://orcid.org/0000-0002-4454-2595"]},"institutions":[{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Wu, Ronghua","raw_affiliation_strings":["Ant Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group","institution_ids":["https://openalex.org/I4405270328"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016394429","display_name":"Qunshan Gu","orcid":"https://orcid.org/0009-0009-5798-8172","observed_orcids":["https://orcid.org/0009-0009-5798-8172"]},"institutions":[{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Gu, Qunshan","raw_affiliation_strings":["Ant Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group","institution_ids":["https://openalex.org/I4405270328"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100645703","display_name":"Yanfeng Wang","orcid":"https://orcid.org/0000-0001-7873-6577","observed_orcids":["https://orcid.org/0000-0001-7873-6577"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Wang, Yanfeng","raw_affiliation_strings":["Ant Group","Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group","institution_ids":["https://openalex.org/I4405270328"]},{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100445125","display_name":"Yu Wang","orcid":"https://orcid.org/0000-0001-9500-081X","observed_orcids":["https://orcid.org/0000-0001-9500-081X"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Wang, Yu","raw_affiliation_strings":["Ant Group","Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group","institution_ids":["https://openalex.org/I4405270328"]},{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":32,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.6279000043869019,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.6279000043869019,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.37130001187324524,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speech-large-language-models","display_name":"speech large language models","score":0.8130000233650208},{"id":"https://openalex.org/keywords/spoken-dialogue-systems","display_name":"spoken dialogue systems","score":0.7049999833106995},{"id":"https://openalex.org/keywords/conversational-skills","display_name":"conversation skills","score":0.5910000205039978},{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.6309999823570251},{"id":"https://openalex.org/keywords/semantic-quality","display_name":"semantic quality","score":0.6200000047683716},{"id":"https://openalex.org/keywords/acoustic-performance","display_name":"acoustic performance","score":0.777999997138977},{"id":"https://openalex.org/keywords/robustness","display_name":"robustness","score":0.9470000267028809},{"id":"https://openalex.org/keywords/mandarin","display_name":"mandarin","score":0.47999998927116394},{"id":"https://openalex.org/keywords/english-language","display_name":"English language","score":0.7860000133514404}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7462000250816345},{"id":"https://openalex.org/C133378560","wikidata":"https://www.wikidata.org/wiki/Q1753225","display_name":"Paralanguage","level":2,"score":0.7045999765396118},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5612000226974487},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.48739999532699585},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.48399999737739563},{"id":"https://openalex.org/C138954614","wikidata":"https://www.wikidata.org/wiki/Q9192","display_name":"Mandarin Chinese","level":2,"score":0.43619999289512634},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.415800005197525},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.4124999940395355},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.3718000054359436},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.36160001158714294},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3497999906539917},{"id":"https://openalex.org/C2776230583","wikidata":"https://www.wikidata.org/wiki/Q1322198","display_name":"Spoken language","level":2,"score":0.3434000015258789},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34119999408721924},{"id":"https://openalex.org/C504749915","wikidata":"https://www.wikidata.org/wiki/Q9010971","display_name":"Speech technology","level":3,"score":0.314300000667572},{"id":"https://openalex.org/C91863865","wikidata":"https://www.wikidata.org/wiki/Q4349497","display_name":"Speech corpus","level":3,"score":0.2897000014781952},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.2842999994754791},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.2689000070095062},{"id":"https://openalex.org/C2992801459","wikidata":"https://www.wikidata.org/wiki/Q223642","display_name":"Interpersonal interaction","level":2,"score":0.26759999990463257},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.26019999384880066},{"id":"https://openalex.org/C19768560","wikidata":"https://www.wikidata.org/wiki/Q320727","display_name":"Dependency (UML)","level":2,"score":0.2567000091075897},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.25440001487731934},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2515999972820282}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2505.15727","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.15727","pdf_url":"https://arxiv.org/pdf/2505.15727","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.15727","doi":"https://doi.org/10.48550/arxiv.2505.15727","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.15727","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2505.15727","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.15727","pdf_url":"https://arxiv.org/pdf/2505.15727","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415329203.pdf","grobid_xml":"https://content.openalex.org/works/W4415329203.grobid-xml"},"referenced_works_count":47,"referenced_works":["https://openalex.org/W1982728343","https://openalex.org/W2030931454","https://openalex.org/W2164240509","https://openalex.org/W2219249508","https://openalex.org/W2243752967","https://openalex.org/W2252136820","https://openalex.org/W2612431505","https://openalex.org/W2738674657","https://openalex.org/W2891575196","https://openalex.org/W2963995027","https://openalex.org/W3121904249","https://openalex.org/W3210904070","https://openalex.org/W4224313995","https://openalex.org/W4225956675","https://openalex.org/W4385571157","https://openalex.org/W4388718089","https://openalex.org/W4393178509","https://openalex.org/W4399425629","https://openalex.org/W4400434308","https://openalex.org/W4400518537","https://openalex.org/W4400717319","https://openalex.org/W4402669711","https://openalex.org/W4402671584","https://openalex.org/W4402672064","https://openalex.org/W4402684170","https://openalex.org/W4402684226","https://openalex.org/W4403556012","https://openalex.org/W4403576555","https://openalex.org/W4403624688","https://openalex.org/W4403883071","https://openalex.org/W4404261856","https://openalex.org/W4404390495","https://openalex.org/W4404649925","https://openalex.org/W4405057125","https://openalex.org/W4405433404","https://openalex.org/W4405714880","https://openalex.org/W4406880404","https://openalex.org/W4407814496","https://openalex.org/W4408354728","https://openalex.org/W4409051416","https://openalex.org/W4412944876","https://openalex.org/W4415797422","https://openalex.org/W4416034301","https://openalex.org/W4416125252","https://openalex.org/W4416381162","https://openalex.org/W6948212704","https://openalex.org/W7133202489"],"related_works":["https://openalex.org/W4404261856","https://openalex.org/W4415433003","https://openalex.org/W6929476346","https://openalex.org/W7212185385","https://openalex.org/W4417101201","https://openalex.org/W4417101690","https://openalex.org/W4416034301","https://openalex.org/W7128775207","https://openalex.org/W4415232713"],"abstract_inverted_index":{"Speech":[0],"large":[1],"language":[2],"models":[3,39,109],"(SpeechLLMs)":[4],"have":[5],"extended":[6],"human-machine":[7],"interactions":[8],"from":[9],"the":[10,14,49,73,111,119],"text":[11],"modality":[12],"to":[13,71],"dynamic":[15],"speech":[16,37,74,126],"domain.":[17],"Spoken":[18],"dialogues":[19],"convey":[20],"diverse":[21],"information,":[22],"including":[23],"semantic":[24,93],"concepts,":[25],"acoustic":[26,95],"variations,":[27],"paralanguage":[28],"cues,":[29],"and":[30,45,86,99,117],"environmental":[31],"context.":[32],"However,":[33],"existing":[34],"evaluations":[35],"of":[36,51,58,83],"interaction":[38],"lack":[40],"instances":[41,82],"mimicking":[42],"real":[43],"scenarios":[44],"predominantly":[46],"focus":[47],"on":[48,106],"performance":[50],"distinct":[52],"aspects,":[53],"lacking":[54],"a":[55],"comprehensive":[56],"comparison":[57],"critical":[59],"capabilities":[60],"between":[61],"current":[62,115],"routines.":[63],"To":[64],"address":[65],"this":[66],"gap,":[67],"we":[68],"propose":[69],"VocalBench":[70],"assess":[72],"conversational":[75,97],"abilities,":[76,98],"comprising":[77],"around":[78],"24k":[79],"carefully":[80],"curated":[81],"both":[84],"English":[85],"Mandarin":[87],"across":[88],"four":[89],"key":[90],"dimensions":[91],"-":[92],"quality,":[94],"performance,":[96],"robustness,":[100],"covering":[101],"14":[102],"user-oriented":[103],"characters.":[104],"Experiments":[105],"27":[107],"mainstream":[108],"reveal":[110],"common":[112],"challenges":[113],"for":[114,121],"routes,":[116],"highlight":[118],"need":[120],"new":[122],"insights":[123],"into":[124],"next-generation":[125],"interactive":[127],"systems.":[128]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4415329203","counts_by_year":[{"year":2026,"cited_by_count":24},{"year":2025,"cited_by_count":8}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2025-10-19T00:00:00"}
