{"id":"https://openalex.org/W7214733748","doi":"https://doi.org/10.48550/arxiv.2609.32016","title":"VoiceNet: Fine-Grained Voice Understanding Beyond Emotion at Scale","display_name":"VoiceNet: Fine-Grained Voice Understanding Beyond Emotion at Scale","publication_year":2026,"publication_date":"2026-09-25","ids":{"openalex":"https://openalex.org/W7214733748","doi":"https://doi.org/10.48550/arxiv.2609.32016"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.32016","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.32016","pdf_url":"https://arxiv.org/pdf/2609.32016","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.32016","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109076838","display_name":"Christoph G. Schuhmann","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schuhmann, Christoph","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051629090","display_name":"Robert Kaczmarczyk","orcid":"https://orcid.org/0000-0002-8570-1601","observed_orcids":["https://orcid.org/0000-0002-8570-1601"]},"institutions":[{"id":"https://openalex.org/I171892758","display_name":"Forschungszentrum J\u00fclich","ror":"https://ror.org/02nv7yv05","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I171892758"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Kaczmarczyk, Robert","raw_affiliation_strings":["Scalable Learning & Multi-Purpose AI (SLAMPAI) Lab , Forschungszentrum J\u00fclich GmbH"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Scalable Learning & Multi-Purpose AI (SLAMPAI) Lab , Forschungszentrum J\u00fclich GmbH","institution_ids":["https://openalex.org/I171892758"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028535577","display_name":"Gollam Rabby","orcid":"https://orcid.org/0000-0002-1212-0101","observed_orcids":["https://orcid.org/0000-0002-1212-0101"]},"institutions":[{"id":"https://openalex.org/I4210136150","display_name":"L3S Research Center","ror":"https://ror.org/039t4wk02","country_code":"DE","type":"facility","lineage":["https://openalex.org/I114112103","https://openalex.org/I4210136150","https://openalex.org/I94509681"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Rabby, Gollam","raw_affiliation_strings":["L3S Research Center"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"L3S Research Center","institution_ids":["https://openalex.org/I4210136150"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038205500","display_name":"Felix Friedrich","orcid":"https://orcid.org/0000-0001-8387-793X","observed_orcids":["https://orcid.org/0000-0001-8387-793X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Friedrich, Felix","raw_affiliation_strings":["Black Forest Labs"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Black Forest Labs","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102620284","display_name":"Maurice Kraus","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technische Universit\u00e4t Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Kraus, Maurice","raw_affiliation_strings":["Computer Science Department , TU Darmstadt"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department , TU Darmstadt","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005923237","display_name":"Gijs Wijngaard","orcid":"https://orcid.org/0009-0002-3875-1232","observed_orcids":["https://orcid.org/0009-0002-3875-1232"]},"institutions":[{"id":"https://openalex.org/I171892758","display_name":"Forschungszentrum J\u00fclich","ror":"https://ror.org/02nv7yv05","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I171892758"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Wijngaard, Gijs","raw_affiliation_strings":["Scalable Learning & Multi-Purpose AI (SLAMPAI) Lab , Forschungszentrum J\u00fclich GmbH"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Scalable Learning & Multi-Purpose AI (SLAMPAI) Lab , Forschungszentrum J\u00fclich GmbH","institution_ids":["https://openalex.org/I171892758"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119938190","display_name":"Kourosh Nadi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nadi, Kourosh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111160371","display_name":"Huu Nguyen","orcid":"https://orcid.org/0000-0002-0466-9532","observed_orcids":["https://orcid.org/0000-0002-0466-9532"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Huu","raw_affiliation_strings":["Ontocord.AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ontocord.AI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037636074","display_name":"Kristian Kersting","orcid":"https://orcid.org/0000-0002-2873-9152","observed_orcids":["https://orcid.org/0000-0002-2873-9152"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kersting, Kristian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5071765665","display_name":"S\u00f6ren Auer","orcid":"https://orcid.org/0000-0002-0698-2864","observed_orcids":["https://orcid.org/0000-0002-0698-2864"]},"institutions":[{"id":"https://openalex.org/I114112103","display_name":"Leibniz University Hannover","ror":"https://ror.org/0304hq317","country_code":"DE","type":"education","lineage":["https://openalex.org/I114112103"]},{"id":"https://openalex.org/I4210136150","display_name":"L3S Research Center","ror":"https://ror.org/039t4wk02","country_code":"DE","type":"facility","lineage":["https://openalex.org/I114112103","https://openalex.org/I4210136150","https://openalex.org/I94509681"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Auer, S\u00f6ren","raw_affiliation_strings":["L3S Research Center","Leibniz Universit\u00e4t Hannover"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"L3S Research Center","institution_ids":["https://openalex.org/I4210136150"]},{"raw_affiliation_string":"Leibniz Universit\u00e4t Hannover","institution_ids":["https://openalex.org/I114112103"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.5967000126838684,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.5967000126838684,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.37229999899864197,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.010999999940395355,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"emotion recognition","score":0.4880000054836273},{"id":"https://openalex.org/keywords/vocal-registers","display_name":"vocal registers","score":0.3779999911785126},{"id":"https://openalex.org/keywords/vocal-tension","display_name":"vocal tension","score":0.6029999852180481},{"id":"https://openalex.org/keywords/breathiness","display_name":"breathiness","score":0.5989999771118164},{"id":"https://openalex.org/keywords/in-the-wild-speech","display_name":"in-the-wild speech","score":0.6639999747276306}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6646999716758728},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.555400013923645},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5047000050544739},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.4860999882221222},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.47760000824928284},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45320001244544983},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.3880000114440918},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3824999928474426},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.38100001215934753},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3553999960422516},{"id":"https://openalex.org/C206310091","wikidata":"https://www.wikidata.org/wiki/Q750859","display_name":"Emotion classification","level":2,"score":0.3481999933719635},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32109999656677246},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.32089999318122864},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.29739999771118164},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.28110000491142273},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.25130000710487366}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.32016","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.32016","pdf_url":"https://arxiv.org/pdf/2609.32016","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.32016","doi":"https://doi.org/10.48550/arxiv.2609.32016","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.32016","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.32016","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.32016","pdf_url":"https://arxiv.org/pdf/2609.32016","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321114","display_name":"Bundesministerium f\u00fcr Bildung und Forschung","ror":"https://ror.org/04pz7b180"},{"id":"https://openalex.org/F4320323384","display_name":"Technische Universit\u00e4t Darmstadt","ror":"https://ror.org/05n911h24"},{"id":"https://openalex.org/F4320334763","display_name":"Leibniz-Gemeinschaft","ror":"https://ror.org/01n6r0e97"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214733748.pdf","grobid_xml":"https://content.openalex.org/works/W7214733748.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4417308701","https://openalex.org/W7204865072","https://openalex.org/W7135405773","https://openalex.org/W2810914326","https://openalex.org/W7214206609","https://openalex.org/W7125506549","https://openalex.org/W2332095503","https://openalex.org/W7084103406","https://openalex.org/W4415432927","https://openalex.org/W4385822669"],"abstract_inverted_index":{"Expressive":[0],"speech":[1,6,198],"synthesis":[2],"has":[3,55],"outpaced":[4],"expressive":[5],"perception:":[7],"systems":[8,176],"now":[9],"render":[10],"fine-grained":[11],"vocal":[12,81],"performances":[13],"that":[14,202],"no":[15],"public":[16],"benchmark":[17,45],"can":[18],"score.":[19],"Most":[20],"benchmarks":[21],"for":[22,46,120,129,219,229],"this":[23,115,220],"inverse":[24],"problem":[25],"stop":[26],"at":[27],"six":[28],"to":[29],"nine":[30],"basic":[31],"emotion":[32,173],"categories,":[33],"largely":[34],"on":[35,50,114,141],"acted":[36],"speech.":[37,53],"This":[38],"paper":[39,87],"introduces":[40],"VoiceNet,":[41,222],"a":[42,60,71,99,117,126,161,216],"human-annotated":[43],"representation-level":[44,185],"voice":[47],"performance":[48],"understanding":[49],"permissively-licensed":[51],"in-the-wild":[52],"VoiceNet":[54,183],"two":[56],"subsets:":[57],"VoiceNet-Emo":[58],"applies":[59],"40-emotion":[61],"taxonomy":[62],"with":[63,98,149,158],"three":[64],"expert":[65,152],"ratings":[66],"per":[67],"item,":[68],"and":[69,84,125,188,195,207,224],"VoiceNet-Ext,":[70],"preliminary":[72],"subset,":[73],"scores":[74,184],"57":[75],"talking-style":[76],"attributes":[77],"including":[78],"speaking":[79],"rate,":[80],"tension,":[82],"breathiness,":[83],"register.":[85],"The":[86],"also":[88],"releases":[89],"Emolia,":[90,223],"an":[91,210],"emotion-annotated":[92],"version":[93],"of":[94,170],"the":[95,150,164],"Emilia":[96],"corpus,":[97],"curated":[100],"rebalanced":[101],"subset":[102],"enriched":[103],"by":[104],"dense":[105],"MOSS-Audio":[106],"Thinking":[107],"annotations.":[108],"Two":[109],"voice-text":[110,180],"contrastive":[111],"baselines":[112],"train":[113],"data:":[116],"110M-parameter":[118],"VoiceCLAP-Small":[119],"fast":[121],"large-scale":[122],"data":[123],"filtering":[124,196],"7B":[127],"VoiceCLAP-Large":[128,145],"state-of-the-art":[130],"performance.":[131],"Both":[132],"outperform":[133],"existing":[134],"CLAP":[135],"baselines,":[136],"which":[137],"sit":[138],"near":[139],"chance":[140],"VoiceNet-Emo.":[142],"On":[143],"VoiceNet-Emo,":[144],"aligns":[146],"more":[147],"closely":[148],"aggregate":[151],"consensus":[153],"than":[154,168],"individual":[155],"experts":[156],"agree":[157],"one":[159],"another:":[160],"comparison":[162],"against":[163],"majority":[165],"label":[166],"rather":[167],"evidence":[169],"surpassing":[171],"human":[172],"perception.":[174],"All":[175],"evaluated":[177],"here":[178],"are":[179,226],"embedding":[181],"models:":[182],"attribute":[186],"recognition":[187],"retrieval,":[189],"not":[190],"end-to-end":[191],"spoken-dialogue":[192],"behaviour.":[193],"Clustering":[194],"uncurated":[197],"corpora":[199],"into":[200],"subsets":[201],"span":[203],"diverse":[204],"talking":[205],"styles":[206],"emotions":[208],"remains":[209],"open":[211],"challenge;":[212],"VoiceCLAP":[213,225],"embeddings":[214],"offer":[215],"promising":[217],"tool":[218],"task.":[221],"publicly":[227],"available":[228],"research":[230],"use.":[231]},"counts_by_year":[],"updated_date":"2026-11-01T11:40:01.658920","created_date":"2026-09-30T00:00:00"}