{"id":"https://openalex.org/W4403913178","doi":"https://doi.org/10.1145/3678957.3685729","title":"Multimodal User Enjoyment Detection in Human-Robot Conversation: The Power of Large Language Models","display_name":"Multimodal User Enjoyment Detection in Human-Robot Conversation: The Power of Large Language Models","publication_year":2024,"publication_date":"2024-10-30","ids":{"openalex":"https://openalex.org/W4403913178","doi":"https://doi.org/10.1145/3678957.3685729"},"language":"en","primary_location":{"id":"doi:10.1145/3678957.3685729","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3678957.3685729","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Conference on Multimodel Interaction","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3678957.3685729","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100612354","display_name":"Andr\u00e9 Pereira","orcid":"https://orcid.org/0000-0003-2428-0468"},"institutions":[{"id":"https://openalex.org/I86987016","display_name":"KTH Royal Institute of Technology","ror":"https://ror.org/026vcq606","country_code":"SE","type":"education","lineage":["https://openalex.org/I86987016"]}],"countries":["SE"],"is_corresponding":true,"raw_author_name":"Andre Pereira","raw_affiliation_strings":["KTH Royal Institute of Technology, Sweden"],"affiliations":[{"raw_affiliation_string":"KTH Royal Institute of Technology, Sweden","institution_ids":["https://openalex.org/I86987016"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026854512","display_name":"\u013dubo\u0161 Marcinek","orcid":"https://orcid.org/0000-0002-1001-6415"},"institutions":[{"id":"https://openalex.org/I86987016","display_name":"KTH Royal Institute of Technology","ror":"https://ror.org/026vcq606","country_code":"SE","type":"education","lineage":["https://openalex.org/I86987016"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Lubos Marcinek","raw_affiliation_strings":["KTH Royal Institute of Technology, Sweden"],"affiliations":[{"raw_affiliation_string":"KTH Royal Institute of Technology, Sweden","institution_ids":["https://openalex.org/I86987016"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093251775","display_name":"J\u016bra Miniota","orcid":null},"institutions":[{"id":"https://openalex.org/I86987016","display_name":"KTH Royal Institute of Technology","ror":"https://ror.org/026vcq606","country_code":"SE","type":"education","lineage":["https://openalex.org/I86987016"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Jura Miniota","raw_affiliation_strings":["KTH Royal Institute of Technology, Sweden"],"affiliations":[{"raw_affiliation_string":"KTH Royal Institute of Technology, Sweden","institution_ids":["https://openalex.org/I86987016"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077914073","display_name":"Sofia Thunberg","orcid":"https://orcid.org/0000-0002-7556-5079"},"institutions":[{"id":"https://openalex.org/I102134673","display_name":"Link\u00f6ping University","ror":"https://ror.org/05ynxx418","country_code":"SE","type":"education","lineage":["https://openalex.org/I102134673"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Sofia Thunberg","raw_affiliation_strings":["Link\u00f6ping University, Sweden"],"affiliations":[{"raw_affiliation_string":"Link\u00f6ping University, Sweden","institution_ids":["https://openalex.org/I102134673"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062250473","display_name":"Erik Lagerstedt","orcid":"https://orcid.org/0000-0002-8937-8063"},"institutions":[{"id":"https://openalex.org/I205158640","display_name":"University of Sk\u00f6vde","ror":"https://ror.org/051mrsz47","country_code":"SE","type":"education","lineage":["https://openalex.org/I205158640"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Erik Lagerstedt","raw_affiliation_strings":["School of Informatics, University of Sk\u00f6vde, Sweden"],"affiliations":[{"raw_affiliation_string":"School of Informatics, University of Sk\u00f6vde, Sweden","institution_ids":["https://openalex.org/I205158640"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015417808","display_name":"Joakim Gustafson","orcid":"https://orcid.org/0000-0002-0397-6442"},"institutions":[{"id":"https://openalex.org/I86987016","display_name":"KTH Royal Institute of Technology","ror":"https://ror.org/026vcq606","country_code":"SE","type":"education","lineage":["https://openalex.org/I86987016"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Joakim Gustafson","raw_affiliation_strings":["KTH Royal Institute of Technology, Sweden"],"affiliations":[{"raw_affiliation_string":"KTH Royal Institute of Technology, Sweden","institution_ids":["https://openalex.org/I86987016"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091225228","display_name":"Gabriel Skantze","orcid":"https://orcid.org/0000-0002-8579-1790"},"institutions":[{"id":"https://openalex.org/I86987016","display_name":"KTH Royal Institute of Technology","ror":"https://ror.org/026vcq606","country_code":"SE","type":"education","lineage":["https://openalex.org/I86987016"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Gabriel Skantze","raw_affiliation_strings":["KTH Royal Institute of Technology, Sweden"],"affiliations":[{"raw_affiliation_string":"KTH Royal Institute of Technology, Sweden","institution_ids":["https://openalex.org/I86987016"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091756638","display_name":"Bahar Irfan","orcid":"https://orcid.org/0000-0002-7983-079X"},"institutions":[{"id":"https://openalex.org/I86987016","display_name":"KTH Royal Institute of Technology","ror":"https://ror.org/026vcq606","country_code":"SE","type":"education","lineage":["https://openalex.org/I86987016"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Bahar Irfan","raw_affiliation_strings":["KTH Royal Institute of Technology, Sweden"],"affiliations":[{"raw_affiliation_string":"KTH Royal Institute of Technology, Sweden","institution_ids":["https://openalex.org/I86987016"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5100612354"],"corresponding_institution_ids":["https://openalex.org/I86987016"],"apc_list":null,"apc_paid":null,"fwci":4.4111,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.94473925,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"469","last_page":"478"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9896000027656555,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/conversation","display_name":"Conversation","score":0.8450861573219299},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7280179262161255},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.6436040997505188},{"id":"https://openalex.org/keywords/human\u2013robot-interaction","display_name":"Human\u2013robot interaction","score":0.6379035711288452},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.5064908266067505},{"id":"https://openalex.org/keywords/power","display_name":"Power (physics)","score":0.4112863540649414},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3581651449203491},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.33928972482681274},{"id":"https://openalex.org/keywords/communication","display_name":"Communication","score":0.16372349858283997},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.15789416432380676}],"concepts":[{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.8450861573219299},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7280179262161255},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.6436040997505188},{"id":"https://openalex.org/C145460709","wikidata":"https://www.wikidata.org/wiki/Q859951","display_name":"Human\u2013robot interaction","level":3,"score":0.6379035711288452},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5064908266067505},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.4112863540649414},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3581651449203491},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33928972482681274},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.16372349858283997},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.15789416432380676},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3678957.3685729","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3678957.3685729","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Conference on Multimodel Interaction","raw_type":"proceedings-article"},{"id":"pmh:oai:DiVA.org:his-24775","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:his:diva-24775","pdf_url":"https://his.diva-portal.org/smash/get/diva2:1920341/FULLTEXT01","source":{"id":"https://openalex.org/S4306401029","display_name":"University Library of Sk\u00f6vde (University of Sk\u00f6vde)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205158640","host_organization_name":"University of Sk\u00f6vde","host_organization_lineage":["https://openalex.org/I205158640"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:DiVA.org:uu-583623","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:uu:diva-583623","pdf_url":null,"source":{"id":"https://openalex.org/S4306402042","display_name":"Uppsala University Publications (Uppsala University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I123387679","host_organization_name":"Uppsala University","host_organization_lineage":["https://openalex.org/I123387679"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3678957.3685729","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3678957.3685729","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Conference on Multimodel Interaction","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7201223071","display_name":null,"funder_award_id":"2021-05803","funder_id":"https://openalex.org/F4320322581","funder_display_name":"Vetenskapsr\u00e5det"}],"funders":[{"id":"https://openalex.org/F4320322581","display_name":"Vetenskapsr\u00e5det","ror":"https://ror.org/03zttf063"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W1504376199","https://openalex.org/W1982332938","https://openalex.org/W2000436309","https://openalex.org/W2029538510","https://openalex.org/W2030707874","https://openalex.org/W2033921274","https://openalex.org/W2034527451","https://openalex.org/W2137577228","https://openalex.org/W2138672357","https://openalex.org/W2186265640","https://openalex.org/W2295598076","https://openalex.org/W2478911292","https://openalex.org/W2550497374","https://openalex.org/W2613959975","https://openalex.org/W2787369546","https://openalex.org/W2790179348","https://openalex.org/W2887390696","https://openalex.org/W2980906155","https://openalex.org/W2996400041","https://openalex.org/W3014773865","https://openalex.org/W3115984996","https://openalex.org/W3138049487","https://openalex.org/W3142113686","https://openalex.org/W3205052962","https://openalex.org/W4246535998","https://openalex.org/W4283170666","https://openalex.org/W4312752637","https://openalex.org/W4323537006","https://openalex.org/W4323667155","https://openalex.org/W4389010438","https://openalex.org/W4389636360","https://openalex.org/W4392633583","https://openalex.org/W4392633612","https://openalex.org/W4395025834"],"related_works":["https://openalex.org/W2765333006","https://openalex.org/W2978665606","https://openalex.org/W4287179229","https://openalex.org/W3205513966","https://openalex.org/W3120459843","https://openalex.org/W4366547574","https://openalex.org/W3200191727","https://openalex.org/W189465620","https://openalex.org/W4366818884","https://openalex.org/W2248308732"],"abstract_inverted_index":{"Enjoyment":[0],"is":[1,19],"a":[2,27,32,79],"crucial":[3],"yet":[4],"complex":[5],"indicator":[6],"of":[7,88,103,117,133],"positive":[8],"user":[9,74,86,150],"experience":[10],"in":[11,72,122,126],"Human-Robot":[12],"Interaction":[13],"(HRI).":[14],"While":[15,94],"manual":[16],"enjoyment":[17,37,89,137],"annotation":[18,38],"feasible,":[20],"developing":[21],"reliable":[22],"automatic":[23,36],"detection":[24],"methods":[25],"remains":[26],"challenge.":[28],"This":[29,128],"paper":[30],"investigates":[31],"multimodal":[33,59,95],"approach":[34],"to":[35,69,91],"for":[39,135,142],"HRI":[40],"conversations,":[41],"leveraging":[42],"large":[43],"language":[44],"models":[45],"(LLMs),":[46],"visual,":[47],"audio,":[48],"and":[49,58,85,113,119],"temporal":[50],"cues.":[51],"Our":[52],"findings":[53],"demonstrate":[54],"that":[55,146],"both":[56],"text-only":[57],"LLMs":[60,134],"with":[61],"carefully":[62],"designed":[63],"prompts":[64],"can":[65,147],"achieve":[66],"performance":[67,105],"comparable":[68],"human":[70,92,111],"annotators":[71,112],"detecting":[73,123],"enjoyment.":[75,127],"Furthermore,":[76],"results":[77],"reveal":[78],"stronger":[80],"alignment":[81],"between":[82],"LLM-based":[83],"annotations":[84],"self-reports":[87],"compared":[90],"annotators.":[93],"supervised":[96],"learning":[97],"techniques":[98],"did":[99],"not":[100],"improve":[101],"all":[102],"our":[104],"metrics,":[106],"they":[107],"could":[108],"successfully":[109],"replicate":[110],"highlighted":[114],"the":[115,131,140],"importance":[116],"visual":[118],"audio":[120],"cues":[121],"subtle":[124],"shifts":[125],"research":[129],"demonstrates":[130],"potential":[132],"real-time":[136],"detection,":[138],"paving":[139],"way":[141],"adaptive":[143],"companion":[144],"robots":[145],"dynamically":[148],"enhance":[149],"experiences.":[151]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}