{"id":"https://openalex.org/W4392931281","doi":"https://doi.org/10.1109/icassp48485.2024.10446933","title":"Paralinguistics-Enhanced Large Language Modeling of Spoken Dialogue","display_name":"Paralinguistics-Enhanced Large Language Modeling of Spoken Dialogue","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4392931281","doi":"https://doi.org/10.1109/icassp48485.2024.10446933"},"language":"en","primary_location":{"id":"doi:10.1109/icassp48485.2024.10446933","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10446933","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108124525","display_name":"Guan-Ting Lin","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]},{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW","US"],"is_corresponding":false,"raw_author_name":"Guan-Ting Lin","raw_affiliation_strings":["Amazon Alexa AI,USA","National Taiwan University, Taiwan","Amazon Alexa AI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI,USA","institution_ids":["https://openalex.org/I1311688040"]},{"raw_affiliation_string":"National Taiwan University, Taiwan","institution_ids":["https://openalex.org/I16733864"]},{"raw_affiliation_string":"Amazon Alexa AI, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088032905","display_name":"Prashanth Gurunath Shivakumar","orcid":"https://orcid.org/0000-0003-1632-3309"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Prashanth Gurunath Shivakumar","raw_affiliation_strings":["Amazon Alexa AI,USA","Amazon Alexa AI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI,USA","institution_ids":["https://openalex.org/I1311688040"]},{"raw_affiliation_string":"Amazon Alexa AI, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014054572","display_name":"Ankur Gandhe","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ankur Gandhe","raw_affiliation_strings":["Amazon Alexa AI,USA","Amazon Alexa AI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI,USA","institution_ids":["https://openalex.org/I1311688040"]},{"raw_affiliation_string":"Amazon Alexa AI, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020376803","display_name":"Chao-Han Huck Yang","orcid":"https://orcid.org/0000-0003-2879-8811"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chao-Han Huck Yang","raw_affiliation_strings":["Amazon Alexa AI,USA","Amazon Alexa AI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI,USA","institution_ids":["https://openalex.org/I1311688040"]},{"raw_affiliation_string":"Amazon Alexa AI, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022646825","display_name":"Yile Gu","orcid":"https://orcid.org/0000-0003-3662-5056"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yile Gu","raw_affiliation_strings":["Amazon Alexa AI,USA","Amazon Alexa AI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI,USA","institution_ids":["https://openalex.org/I1311688040"]},{"raw_affiliation_string":"Amazon Alexa AI, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102095407","display_name":"Shalini Ghosh","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shalini Ghosh","raw_affiliation_strings":["Amazon Alexa AI,USA","Amazon Alexa AI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI,USA","institution_ids":["https://openalex.org/I1311688040"]},{"raw_affiliation_string":"Amazon Alexa AI, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060979948","display_name":"Andreas Stolcke","orcid":"https://orcid.org/0000-0002-9925-905X"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andreas Stolcke","raw_affiliation_strings":["Amazon Alexa AI,USA","Amazon Alexa AI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI,USA","institution_ids":["https://openalex.org/I1311688040"]},{"raw_affiliation_string":"Amazon Alexa AI, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040508737","display_name":"Hung-yi Lee","orcid":"https://orcid.org/0000-0002-9654-5747"},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Hung-Yi Lee","raw_affiliation_strings":["National Taiwan University,Taiwan","National Taiwan University, Taiwan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Taiwan University,Taiwan","institution_ids":["https://openalex.org/I16733864"]},{"raw_affiliation_string":"National Taiwan University, Taiwan","institution_ids":["https://openalex.org/I16733864"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5109934165","display_name":"Ivan Bulyko","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ivan Bulyko","raw_affiliation_strings":["Amazon Alexa AI,USA","Amazon Alexa AI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI,USA","institution_ids":["https://openalex.org/I1311688040"]},{"raw_affiliation_string":"Amazon Alexa AI, USA","institution_ids":["https://openalex.org/I1311688040"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"10316","last_page":"10320"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/paralanguage","display_name":"Paralanguage","score":0.9778822660446167},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7856195569038391},{"id":"https://openalex.org/keywords/spoken-language","display_name":"Spoken language","score":0.6363906264305115},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5860456228256226},{"id":"https://openalex.org/keywords/conversation","display_name":"Conversation","score":0.557500422000885},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.48192691802978516},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.44983160495758057},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.4494546949863434},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"Natural language understanding","score":0.44255369901657104},{"id":"https://openalex.org/keywords/human-multitasking","display_name":"Human multitasking","score":0.4251851737499237},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.3919069766998291},{"id":"https://openalex.org/keywords/communication","display_name":"Communication","score":0.13219672441482544},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.11944898962974548},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.11006864905357361}],"concepts":[{"id":"https://openalex.org/C133378560","wikidata":"https://www.wikidata.org/wiki/Q1753225","display_name":"Paralanguage","level":2,"score":0.9778822660446167},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7856195569038391},{"id":"https://openalex.org/C2776230583","wikidata":"https://www.wikidata.org/wiki/Q1322198","display_name":"Spoken language","level":2,"score":0.6363906264305115},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5860456228256226},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.557500422000885},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48192691802978516},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.44983160495758057},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4494546949863434},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.44255369901657104},{"id":"https://openalex.org/C107418235","wikidata":"https://www.wikidata.org/wiki/Q1520565","display_name":"Human multitasking","level":2,"score":0.4251851737499237},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3919069766998291},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.13219672441482544},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.11944898962974548},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.11006864905357361},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp48485.2024.10446933","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10446933","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5699999928474426,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W2128970689","https://openalex.org/W2151083697","https://openalex.org/W2166637769","https://openalex.org/W2548264631","https://openalex.org/W2586286573","https://openalex.org/W2766833041","https://openalex.org/W2897636448","https://openalex.org/W2936162287","https://openalex.org/W2952088495","https://openalex.org/W2968228919","https://openalex.org/W3015489952","https://openalex.org/W3036601975","https://openalex.org/W3048664667","https://openalex.org/W3095607145","https://openalex.org/W3133903425","https://openalex.org/W3153025627","https://openalex.org/W3174716116","https://openalex.org/W3193377986","https://openalex.org/W3197236059","https://openalex.org/W3198217962","https://openalex.org/W3198771897","https://openalex.org/W4226422436","https://openalex.org/W4285273040","https://openalex.org/W4293714393","https://openalex.org/W4307680525","https://openalex.org/W4319862479","https://openalex.org/W4319862652","https://openalex.org/W4366559955","https://openalex.org/W4381786045","https://openalex.org/W4381827575","https://openalex.org/W4385823311","https://openalex.org/W4389524500","https://openalex.org/W4394671563","https://openalex.org/W6763494226","https://openalex.org/W6778490555","https://openalex.org/W6780218876","https://openalex.org/W6790356757","https://openalex.org/W6791303579","https://openalex.org/W6804073865","https://openalex.org/W6851678396","https://openalex.org/W6853998256"],"related_works":["https://openalex.org/W2375682379","https://openalex.org/W163288733","https://openalex.org/W3174008653","https://openalex.org/W2765804957","https://openalex.org/W2962716343","https://openalex.org/W4288099861","https://openalex.org/W2893411096","https://openalex.org/W4224919006","https://openalex.org/W43702919","https://openalex.org/W4213400064"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"superior":[6],"abilities":[7],"in":[8,104,187],"tasks":[9,103],"such":[10,24,42],"as":[11,25,90,132,136],"chatting,":[12],"reasoning,":[13],"and":[14,28,62,71,87,116,156,164,173,185,194],"question-answering.":[15],"However,":[16],"standard":[17],"LLMs":[18],"may":[19],"ignore":[20],"crucial":[21],"paralinguistic":[22,72,88,109,113,134],"information,":[23],"sentiment,":[26],"emotion,":[27],"speaking":[29],"style,":[30],"which":[31],"are":[32],"essential":[33],"for":[34],"achieving":[35],"natural,":[36],"human-like":[37],"spoken":[38,75,138],"conversation,":[39],"especially":[40],"when":[41],"information":[43],"is":[44],"conveyed":[45],"by":[46],"acoustic":[47],"cues.":[48],"We":[49,123],"therefore":[50],"propose":[51],"Paralinguistics-enhanced":[52],"Generative":[53],"Pretrained":[54],"Transformer":[55],"(ParalinGPT),":[56],"an":[57],"LLM":[58],"that":[59],"utilizes":[60],"text":[61,118,171,196],"speech":[63,85,165],"modalities":[64],"to":[65],"better":[66],"model":[67,78],"the":[68,80,105,125,133,144],"linguistic":[69],"content":[70],"attributes":[73,89],"of":[74,83,107,182],"dialogue.":[76],"The":[77],"takes":[79],"conversational":[81,162],"context":[82,163],"text,":[84],"embeddings,":[86],"input":[91],"prompts":[92],"within":[93],"a":[94],"serialized":[95,146],"multitasking":[96,147],"multimodal":[97],"framework.":[98],"Specifically,":[99],"our":[100,137],"framework":[101,178],"serializes":[102],"order":[106],"current":[108,155,188],"attribute":[110,114],"prediction,":[111,115],"response":[112,117,157,170,191,195],"generation":[119,172],"with":[120],"autoregressive":[121],"conditioning.":[122],"utilize":[124],"Switchboard-1":[126],"corpus,":[127],"including":[128],"its":[129],"sentiment":[130,158,174,189,192],"labels":[131],"attribute,":[135],"dialogue":[139],"dataset.":[140],"Experimental":[141],"results":[142],"indicate":[143],"proposed":[145,177],"method":[148],"outperforms":[149],"typical":[150],"sequence":[151],"classification":[152],"techniques":[153],"on":[154],"classification.":[159],"Furthermore,":[160],"leveraging":[161],"embeddings":[166],"significantly":[167],"improves":[168],"both":[169],"prediction.":[175],"Our":[176],"achieves":[179],"relative":[180],"improvements":[181],"6.7%,":[183],"12.0%,":[184],"3.5%":[186],"accuracy,":[190,193],"BLEU":[197],"score,":[198],"respectively.":[199]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}