{"id":"https://openalex.org/W4413948551","doi":"https://doi.org/10.1109/mra.2025.3543957","title":"Natural Multimodal Fusion-Based Human\u2013Robot Interaction: Application With Voice and Deictic Posture via Large Language Model","display_name":"Natural Multimodal Fusion-Based Human\u2013Robot Interaction: Application With Voice and Deictic Posture via Large Language Model","publication_year":2025,"publication_date":"2025-03-05","ids":{"openalex":"https://openalex.org/W4413948551","doi":"https://doi.org/10.1109/mra.2025.3543957"},"language":"en","primary_location":{"id":"doi:10.1109/mra.2025.3543957","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mra.2025.3543957","pdf_url":null,"source":{"id":"https://openalex.org/S132261465","display_name":"IEEE Robotics & Automation Magazine","issn_l":"1070-9932","issn":["1070-9932","1558-223X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics & Automation Magazine","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002297490","display_name":"Yuzhi Lai","orcid":"https://orcid.org/0009-0002-9976-7422"},"institutions":[{"id":"https://openalex.org/I143910747","display_name":"TH Bingen University of Applied Sciences","ror":"https://ror.org/01pxkj057","country_code":"DE","type":"education","lineage":["https://openalex.org/I143910747"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Yuzhi Lai","raw_affiliation_strings":["University of Tübingen, Tübingen, Germany"],"raw_orcid":"https://orcid.org/0009-0002-9976-7422","affiliations":[{"raw_affiliation_string":"University of Tübingen, Tübingen, Germany","institution_ids":["https://openalex.org/I143910747"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026848789","display_name":"Shenghai Yuan","orcid":"https://orcid.org/0009-0003-1887-6342"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Shenghai Yuan","raw_affiliation_strings":["Nanyang Technological University, Singapore"],"raw_orcid":"https://orcid.org/0009-0003-1887-6342","affiliations":[{"raw_affiliation_string":"Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5096717934","display_name":"Youssef Nassar","orcid":"https://orcid.org/0009-0007-2867-4030"},"institutions":[{"id":"https://openalex.org/I57589565","display_name":"Reutlingen University","ror":"https://ror.org/00q644y50","country_code":"DE","type":"education","lineage":["https://openalex.org/I57589565"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Youssef Nassar","raw_affiliation_strings":["Reutlingen University, Reutlingen, Germany","University Reutlingen, Reutlingen, Germany"],"raw_orcid":"https://orcid.org/0009-0007-2867-4030","affiliations":[{"raw_affiliation_string":"Reutlingen University, Reutlingen, Germany","institution_ids":["https://openalex.org/I57589565"]},{"raw_affiliation_string":"University Reutlingen, Reutlingen, Germany","institution_ids":["https://openalex.org/I57589565"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007888592","display_name":"Mingyu Fan","orcid":"https://orcid.org/0000-0002-0492-4708"},"institutions":[{"id":"https://openalex.org/I181326427","display_name":"Donghua University","ror":"https://ror.org/035psfh38","country_code":"CN","type":"education","lineage":["https://openalex.org/I181326427"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingyu Fan","raw_affiliation_strings":["Donghua University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-0492-4708","affiliations":[{"raw_affiliation_string":"Donghua University, Shanghai, China","institution_ids":["https://openalex.org/I181326427"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033043372","display_name":"Atmaraaj Gopal","orcid":null},"institutions":[{"id":"https://openalex.org/I4210116723","display_name":"Robotics Research (United States)","ror":"https://ror.org/020w2fr77","country_code":"US","type":"company","lineage":["https://openalex.org/I4210116723"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Atmaraaj Gopal","raw_affiliation_strings":["Neura Robotics, Metzingen, Germany"],"raw_orcid":"https://orcid.org/0009-0002-7578-8384","affiliations":[{"raw_affiliation_string":"Neura Robotics, Metzingen, Germany","institution_ids":["https://openalex.org/I4210116723"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115757057","display_name":"Arihiro Yorita","orcid":null},"institutions":[{"id":"https://openalex.org/I206011266","display_name":"Kwansei Gakuin University","ror":"https://ror.org/02qf2tx24","country_code":"JP","type":"education","lineage":["https://openalex.org/I206011266"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Arihiro Yorita","raw_affiliation_strings":["Kwansei Gakuin University, Hyogo, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kwansei Gakuin University, Hyogo, Japan","institution_ids":["https://openalex.org/I206011266"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074076109","display_name":"Naoyuki Kubota","orcid":"https://orcid.org/0000-0001-8829-037X"},"institutions":[{"id":"https://openalex.org/I69740276","display_name":"Tokyo Metropolitan University","ror":"https://ror.org/00ws30h19","country_code":"JP","type":"education","lineage":["https://openalex.org/I69740276"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Naoyuki Kubota","raw_affiliation_strings":["Tokyo Metropolitan University, Hachioji, Japan"],"raw_orcid":"https://orcid.org/0000-0001-8829-037X","affiliations":[{"raw_affiliation_string":"Tokyo Metropolitan University, Hachioji, Japan","institution_ids":["https://openalex.org/I69740276"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051218329","display_name":"Matthias R\u00e4tsch","orcid":"https://orcid.org/0000-0002-8254-8293"},"institutions":[{"id":"https://openalex.org/I57589565","display_name":"Reutlingen University","ror":"https://ror.org/00q644y50","country_code":"DE","type":"education","lineage":["https://openalex.org/I57589565"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Matthias R\u00e4tsch","raw_affiliation_strings":["Reutlingen University, Reutlingen, Germany","University Reutlingen, Reutlingen, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Reutlingen University, Reutlingen, Germany","institution_ids":["https://openalex.org/I57589565"]},{"raw_affiliation_string":"University Reutlingen, Reutlingen, Germany","institution_ids":["https://openalex.org/I57589565"]}]}],"institutions":[],"countries_distinct_count":5,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":11.6682,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.98807646,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"33","issue":"2","first_page":"29","last_page":"38"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.6712999939918518,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.6712999939918518,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.6689000129699707,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.5864999890327454,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/deixis","display_name":"Deixis","score":0.7989620566368103},{"id":"https://openalex.org/keywords/human\u2013robot-interaction","display_name":"Human\u2013robot interaction","score":0.6572941541671753},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6440150141716003},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.5893228650093079},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5309564471244812},{"id":"https://openalex.org/keywords/multimodal-interaction","display_name":"Multimodal interaction","score":0.5183940529823303},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.5183447599411011},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5026907920837402},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.46616867184638977},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.41393980383872986},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.40612781047821045},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.09135088324546814},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.08295190334320068}],"concepts":[{"id":"https://openalex.org/C13077596","wikidata":"https://www.wikidata.org/wiki/Q340015","display_name":"Deixis","level":2,"score":0.7989620566368103},{"id":"https://openalex.org/C145460709","wikidata":"https://www.wikidata.org/wiki/Q859951","display_name":"Human\u2013robot interaction","level":3,"score":0.6572941541671753},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6440150141716003},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5893228650093079},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5309564471244812},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.5183940529823303},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5183447599411011},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5026907920837402},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.46616867184638977},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.41393980383872986},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.40612781047821045},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.09135088324546814},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.08295190334320068},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/mra.2025.3543957","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mra.2025.3543957","pdf_url":null,"source":{"id":"https://openalex.org/S132261465","display_name":"IEEE Robotics & Automation Magazine","issn_l":"1070-9932","issn":["1070-9932","1558-223X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics & Automation Magazine","raw_type":"journal-article"},{"id":"pmh:oai:dr.ntu.edu.sg:10356/182646","is_oa":false,"landing_page_url":"https://doi.org/10.1109/MRA.2025.3543957","pdf_url":null,"source":{"id":"https://openalex.org/S4306402609","display_name":"DR-NTU (Nanyang Technological University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I172675005","host_organization_name":"Nanyang Technological University","host_organization_lineage":["https://openalex.org/I172675005"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Journal Article"},{"id":"pmh:oai:opus.reutlingen-university.de:5598","is_oa":false,"landing_page_url":"https://publikationen.reutlingen-university.de/frontdoor/index/index/docId/5598","pdf_url":null,"source":{"id":"https://openalex.org/S4306400499","display_name":"Reutlingen University Academic Bibliography (Reutlingen University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I57589565","host_organization_name":"Reutlingen University","host_organization_lineage":["https://openalex.org/I57589565"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3205513966","https://openalex.org/W3120459843","https://openalex.org/W4366547574","https://openalex.org/W4281567410","https://openalex.org/W2955859849","https://openalex.org/W2978665606","https://openalex.org/W2016565202","https://openalex.org/W2160505327","https://openalex.org/W2963761716","https://openalex.org/W2599908883"],"abstract_inverted_index":{"Translating":[0],"human":[1],"intent":[2],"into":[3],"robot":[4,112],"commands":[5,28,105],"is":[6,133],"crucial":[7],"for":[8,31],"the":[9,32,45,74,84,164,168],"future":[10],"of":[11,83,141,158],"service":[12],"robots":[13],"in":[14,156],"an":[15],"aging":[16],"society.":[17],"Existing":[18],"human\u2012robot":[19],"interaction":[20,52],"(HRI)":[21],"systems":[22],"relying":[23],"on":[24,93,135],"gestures":[25],"or":[26,40],"verbal":[27],"are":[29,70,90,123],"impractical":[30],"elderly,":[33],"due":[34],"to":[35,61,78,125],"difficulties":[36],"with":[37,103,138],"complex":[38],"syntax":[39,121],"sign":[41],"language.":[42],"To":[43,162],"address":[44],"challenge,":[46],"this":[47],"article":[48],"introduces":[49],"a":[50,63,80,98,144],"multimodal":[51],"framework":[53],"that":[54],"combines":[55],"voice":[56],"and":[57,86,106,160,167,175],"deictic":[58],"posture":[59],"information":[60],"create":[62],"more":[64],"natural":[65],"HRI":[66,154],"system.":[67],"Visual":[68],"cues":[69],"first":[71],"processed":[72],"by":[73],"object":[75],"detection":[76],"model":[77,101],"gain":[79],"global":[81],"understanding":[82],"environment,":[85],"then":[87],"bounding":[88,110],"boxes":[89],"estimated":[91],"based":[92],"depth":[94],"information.":[95],"By":[96],"using":[97,143],"large":[99],"language":[100],"(LLM)":[102],"voice-to-text":[104],"temporally":[107],"aligned":[108],"selected":[109],"boxes,":[111],"action":[113],"sequences":[114],"can":[115],"be":[116],"generated,":[117],"while":[118],"key":[119],"control":[120],"constraints":[122],"applied":[124],"avoid":[126],"potential":[127],"LLM":[128],"hallucination":[129],"issues.":[130],"The":[131],"system":[132],"evaluated":[134],"real-world":[136],"tasks":[137],"varying":[139],"levels":[140],"complexity,":[142],"Universal":[145],"Robots":[146],"UR3e":[147],"manipulator.":[148],"Our":[149],"method":[150],"demonstrates":[151],"significantly":[152],"better":[153],"performance":[155],"terms":[157],"accuracy":[159],"robustness.":[161],"benefit":[163],"research":[165],"community":[166],"general":[169],"public,":[170],"we":[171],"made":[172],"our":[173],"code":[174],"design":[176],"open":[177],"source.":[178]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":8}],"updated_date":"2026-06-16T07:32:37.131356","created_date":"2025-10-10T00:00:00"}