{"id":"https://openalex.org/W4416183232","doi":"https://doi.org/10.1109/mipr67560.2025.00025","title":"Emotion-Aware Speech Synthesis using Multimodal Deep Learning with Visual and Textual Cues","display_name":"Emotion-Aware Speech Synthesis using Multimodal Deep Learning with Visual and Textual Cues","publication_year":2025,"publication_date":"2025-08-06","ids":{"openalex":"https://openalex.org/W4416183232","doi":"https://doi.org/10.1109/mipr67560.2025.00025"},"language":"en","primary_location":{"id":"doi:10.1109/mipr67560.2025.00025","doi":"https://doi.org/10.1109/mipr67560.2025.00025","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mipr67560.2025.00025","pdf_url":null,"source":{"id":"https://openalex.org/S7407088715","display_name":"IEEE International Conference on Multimedia Information Processing and Retrieval (MIPR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 8th International Conference on Multimedia Information Processing and Retrieval (MIPR)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://repositorio.upt.pt/items/f8b496de-2622-4379-ba96-bbc35dca51bc/request-a-copy?bitstream=4d0fea34-1393-4899-9e9d-3090e39acd78","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093067438","display_name":"Ketan Totlani","orcid":"https://orcid.org/0009-0004-9545-946X","observed_orcids":["https://orcid.org/0009-0004-9545-946X"]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ketan Totlani","raw_affiliation_strings":["Viterbi School of Engineering, University of Southern California,Los Angeles,California,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Viterbi School of Engineering, University of Southern California,Los Angeles,California,USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Smital Patil","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Smital Patil","raw_affiliation_strings":["KIT's College of Engineering,Kolhapur,India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KIT's College of Engineering,Kolhapur,India","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051682322","display_name":"A N Sasikumar","orcid":"https://orcid.org/0009-0005-7103-5678","observed_orcids":["https://orcid.org/0009-0005-7103-5678"]},"institutions":[{"id":"https://openalex.org/I876193797","display_name":"Vellore Institute of Technology University","ror":"https://ror.org/00qzypv28","country_code":"IN","type":"education","lineage":["https://openalex.org/I876193797"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Abhijai Sasikumar","raw_affiliation_strings":["School of Computer Science and Engineering, Vellore Institute of Technology,Vellore,Tamil Nadu,India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Vellore Institute of Technology,Vellore,Tamil Nadu,India","institution_ids":["https://openalex.org/I876193797"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Fernando Moreira","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210153204","display_name":"Universidade Portucalense","ror":"https://ror.org/05wm6p530","country_code":"PT","type":"education","lineage":["https://openalex.org/I4210153204"]},{"id":"https://openalex.org/I60858718","display_name":"University of Aveiro","ror":"https://ror.org/00nt41z93","country_code":"PT","type":"education","lineage":["https://openalex.org/I60858718"]}],"countries":["PT"],"is_corresponding":false,"raw_author_name":"Fernando Moreira","raw_affiliation_strings":["REMIT, IJP, Universidade Portucalense, Porto & IEETA, Universidade de Aveiro,Porto,Portugal,4200-072"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"REMIT, IJP, Universidade Portucalense, Porto & IEETA, Universidade de Aveiro,Porto,Portugal,4200-072","institution_ids":["https://openalex.org/I4210153204","https://openalex.org/I60858718"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5058396473","display_name":"Sachi Nandan Mohanty","orcid":"https://orcid.org/0000-0002-4939-0797","observed_orcids":["https://orcid.org/0000-0002-4939-0797"]},"institutions":[{"id":"https://openalex.org/I4401726783","display_name":"VIT-AP University","ror":"https://ror.org/007v4hf75","country_code":"IN","type":"education","lineage":["https://openalex.org/I4401726783"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Sachi Nandan Mohanty","raw_affiliation_strings":["School of Computer Science and Engineering (SCOPE), VIT-AP University,Amaravati,India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering (SCOPE), VIT-AP University,Amaravati,India","institution_ids":["https://openalex.org/I4401726783"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.7133,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.71654306,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"104","last_page":"108"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.0008999999845400453,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/text-to-speech","display_name":"text-to-speech","score":0.9259999990463257},{"id":"https://openalex.org/keywords/multimodal-deep-learning","display_name":"multimodal deep learning","score":0.8180000185966492},{"id":"https://openalex.org/keywords/tacotron2","display_name":"Tacotron2","score":0.7490000128746033},{"id":"https://openalex.org/keywords/conditional-gan","display_name":"conditional generative adversarial networks","score":0.8159999847412109},{"id":"https://openalex.org/keywords/emotional-prosody","display_name":"emotional prosody","score":0.7570000290870667},{"id":"https://openalex.org/keywords/ravdess-dataset","display_name":"RAVDESS dataset","score":0.8629999756813049},{"id":"https://openalex.org/keywords/expressive-speech","display_name":"expressive speech","score":0.6000000238418579},{"id":"https://openalex.org/keywords/virtual-assistants","display_name":"virtual assistants","score":0.6330000162124634}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7057999968528748},{"id":"https://openalex.org/C542774811","wikidata":"https://www.wikidata.org/wiki/Q10880526","display_name":"Prosody","level":2,"score":0.6855000257492065},{"id":"https://openalex.org/C45273575","wikidata":"https://www.wikidata.org/wiki/Q578970","display_name":"Spectrogram","level":2,"score":0.6007000207901001},{"id":"https://openalex.org/C88485024","wikidata":"https://www.wikidata.org/wiki/Q1054571","display_name":"Cepstrum","level":2,"score":0.5634999871253967},{"id":"https://openalex.org/C14999030","wikidata":"https://www.wikidata.org/wiki/Q16346","display_name":"Speech synthesis","level":2,"score":0.557200014591217},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5325999855995178},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5181999802589417},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5126000046730042},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.40619999170303345},{"id":"https://openalex.org/C126780896","wikidata":"https://www.wikidata.org/wiki/Q899871","display_name":"Distortion (music)","level":4,"score":0.3797000050544739},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3695000112056732},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3677000105381012},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3190999925136566},{"id":"https://openalex.org/C504749915","wikidata":"https://www.wikidata.org/wiki/Q9010971","display_name":"Speech technology","level":3,"score":0.2856000065803528},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.2815999984741211},{"id":"https://openalex.org/C151989614","wikidata":"https://www.wikidata.org/wiki/Q440370","display_name":"Mel-frequency cepstrum","level":3,"score":0.2743000090122223},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.2694000005722046},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2558000087738037},{"id":"https://openalex.org/C2778348673","wikidata":"https://www.wikidata.org/wiki/Q739302","display_name":"Production (economics)","level":2,"score":0.25360000133514404}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/mipr67560.2025.00025","doi":"https://doi.org/10.1109/mipr67560.2025.00025","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mipr67560.2025.00025","pdf_url":null,"source":{"id":"https://openalex.org/S7407088715","display_name":"IEEE International Conference on Multimedia Information Processing and Retrieval (MIPR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 8th International Conference on Multimedia Information Processing and Retrieval (MIPR)","raw_type":"proceedings-article"},{"id":"pmh:oai:repositorio.upt.pt:11328/6769","doi":null,"is_oa":true,"landing_page_url":"https://repositorio.upt.pt/items/f8b496de-2622-4379-ba96-bbc35dca51bc/request-a-copy?bitstream=4d0fea34-1393-4899-9e9d-3090e39acd78","pdf_url":null,"source":{"id":"https://openalex.org/S4306402433","display_name":"Portuguese National Funding Agency for Science, Research and Technology (RCAAP Project by FCT)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference paper"}],"best_oa_location":{"id":"pmh:oai:repositorio.upt.pt:11328/6769","doi":null,"is_oa":true,"landing_page_url":"https://repositorio.upt.pt/items/f8b496de-2622-4379-ba96-bbc35dca51bc/request-a-copy?bitstream=4d0fea34-1393-4899-9e9d-3090e39acd78","pdf_url":null,"source":{"id":"https://openalex.org/S4306402433","display_name":"Portuguese National Funding Agency for Science, Research and Technology (RCAAP Project by FCT)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference paper"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":8,"referenced_works":["https://openalex.org/W4391745456","https://openalex.org/W4391790509","https://openalex.org/W4392904615","https://openalex.org/W4392904630","https://openalex.org/W4392904830","https://openalex.org/W4402111427","https://openalex.org/W4405934939","https://openalex.org/W4409642532"],"related_works":["https://openalex.org/W4415708643","https://openalex.org/W4408345862","https://openalex.org/W4417416919","https://openalex.org/W4404446510","https://openalex.org/W4408985855","https://openalex.org/W7124220177","https://openalex.org/W4392981515","https://openalex.org/W7160378333","https://openalex.org/W4406657572","https://openalex.org/W3197034238"],"abstract_inverted_index":{"Contemporary":[0],"Text-to-Speech":[1],"(TTS)":[2],"technologies":[3],"have":[4],"reached":[5],"incredible":[6],"levels":[7],"of":[8,60,141,168,187],"precision":[9],"in":[10],"producing":[11],"proper":[12],"speech.":[13,80,189],"However,":[14],"the":[15,56,61,84,137,160,185],"output":[16],"is":[17,30,65,97,139,145,170],"often":[18],"emotionless":[19],"and":[20,40,50,89,104,117,132,151,166,172],"robotic":[21],"because":[22],"synthesizing":[23],"emotions":[24],"remains":[25],"a":[26,48,68,98,105],"challenge.":[27],"This":[28,81],"problem":[29],"particularly":[31],"important":[32],"for":[33,101,176],"applications":[34],"like":[35],"virtual":[36],"assistants,":[37],"healthcare":[38],"aides,":[39],"immersive":[41],"voice":[42],"technologies,":[43],"which":[44,96,110],"are":[45],"centered":[46],"around":[47],"person":[49],"where":[51],"emotionally":[52,152],"intelligent":[53],"dialogue":[54],"improves":[55,111],"experience.":[57],"The":[58,119,157],"goal":[59],"research":[62],"presented":[63],"here":[64],"to":[66],"design":[67],"deep":[69],"learning,":[70],"multimodal,":[71],"emotion-aware":[72],"speech":[73,87,143],"synthesis":[74,167],"framework":[75],"that":[76,136,144,162],"would":[77],"generate":[78],"expressive":[79],"study":[82],"utilizes":[83],"RAVDESS":[85],"emotional":[86,112],"dataset":[88],"incorporates":[90],"two":[91],"advanced":[92],"models:":[93],"Tacotron":[94],"2,":[95],"sequence-to-sequence":[99],"model":[100],"spectrogram":[102],"generation,":[103],"Prosody-Guided":[106],"Conditional":[107],"GAN":[108],"(cGAN)":[109],"prosody":[113],"by":[114,123,182],"refining":[115],"pitch":[116],"energy.":[118],"experimental":[120],"evaluation":[121],"provided":[122],"Mean":[124],"Opinion":[125],"Score":[126],"(MOS),":[127],"Mel":[128],"Cepstral":[129],"Distortion":[130],"(MCD),":[131],"F0":[133],"RMSE":[134],"proved":[135],"system":[138],"capable":[140],"generating":[142],"both":[146],"extremely":[147],"natural":[148],"(MOS:":[149],"4.32)":[150],"aligned":[153],"(Emotion":[154],"MOS:":[155],"4.15).":[156],"results":[158],"support":[159],"hypothesis":[161],"combining":[163],"prosodic":[164],"conditioning":[165],"spectrograms":[169],"effective":[171],"opens":[173],"new":[174],"possibilities":[175],"next":[177],"generation":[178,186],"AI":[179],"communication":[180],"systems":[181],"significantly":[183],"enhancing":[184],"emotion-laden":[188]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2025-11-11T00:00:00"}