{"id":"https://openalex.org/W4388617516","doi":"https://doi.org/10.1017/s0263574723001510","title":"Multi-modal interaction with transformers: bridging robots and human with natural language","display_name":"Multi-modal interaction with transformers: bridging robots and human with natural language","publication_year":2023,"publication_date":"2023-11-13","ids":{"openalex":"https://openalex.org/W4388617516","doi":"https://doi.org/10.1017/s0263574723001510"},"language":"en","primary_location":{"id":"doi:10.1017/s0263574723001510","is_oa":false,"landing_page_url":"https://doi.org/10.1017/s0263574723001510","pdf_url":null,"source":{"id":"https://openalex.org/S92163612","display_name":"Robotica","issn_l":"0263-5747","issn":["0263-5747","1469-8668"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Robotica","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5038630321","display_name":"Shaochen Wang","orcid":"https://orcid.org/0000-0002-6985-240X"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaochen Wang","raw_affiliation_strings":["Department of Automation, University of Science and Technology of China, Hefei, 230026, China"],"raw_orcid":"https://orcid.org/0000-0002-6985-240X","affiliations":[{"raw_affiliation_string":"Department of Automation, University of Science and Technology of China, Hefei, 230026, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015310258","display_name":"Zhangli Zhou","orcid":"https://orcid.org/0000-0002-4995-4732"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhangli Zhou","raw_affiliation_strings":["Department of Automation, University of Science and Technology of China, Hefei, 230026, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Automation, University of Science and Technology of China, Hefei, 230026, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100365157","display_name":"Bin Li","orcid":"https://orcid.org/0000-0002-2332-3959"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Li","raw_affiliation_strings":["Department of Electronic and Information Science, University of Science and Technology of China, Hefei, 230026, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electronic and Information Science, University of Science and Technology of China, Hefei, 230026, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100450024","display_name":"Zhijun Li","orcid":"https://orcid.org/0000-0002-3909-488X"},"institutions":[{"id":"https://openalex.org/I116953780","display_name":"Tongji University","ror":"https://ror.org/03rc6as71","country_code":"CN","type":"education","lineage":["https://openalex.org/I116953780"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhijun Li","raw_affiliation_strings":["Institute of Artificial Intelligence, Hefei Comprehensive National Science Center, Hefei, 230031, China","School of Mechanical Engineering, Tongji University, Shanghai, 200092, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Artificial Intelligence, Hefei Comprehensive National Science Center, Hefei, 230031, China","institution_ids":[]},{"raw_affiliation_string":"School of Mechanical Engineering, Tongji University, Shanghai, 200092, China","institution_ids":["https://openalex.org/I116953780"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5018344952","display_name":"Zhen Kan","orcid":"https://orcid.org/0000-0003-2069-9544"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Zhen Kan","raw_affiliation_strings":["Department of Automation, University of Science and Technology of China, Hefei, 230026, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Automation, University of Science and Technology of China, Hefei, 230026, China","institution_ids":["https://openalex.org/I126520041"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5018344952"],"corresponding_institution_ids":["https://openalex.org/I126520041"],"apc_list":{"value":3255,"currency":"USD","value_usd":3255},"apc_paid":null,"fwci":2.0005,"has_fulltext":false,"cited_by_count":21,"citation_normalized_percentile":{"value":0.87730646,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"42","issue":"2","first_page":"415","last_page":"434"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7951211929321289},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.6956478357315063},{"id":"https://openalex.org/keywords/executable","display_name":"Executable","score":0.6378204822540283},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.622437596321106},{"id":"https://openalex.org/keywords/grasp","display_name":"GRASP","score":0.5633985996246338},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.550640881061554},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5378656983375549},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.5144019722938538},{"id":"https://openalex.org/keywords/human\u2013robot-interaction","display_name":"Human\u2013robot interaction","score":0.47885963320732117},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.43469658493995667},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"Natural language understanding","score":0.4332769513130188},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.4101802706718445},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3605367839336395},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3553996682167053},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1642472743988037},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.12985938787460327}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7951211929321289},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.6956478357315063},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.6378204822540283},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.622437596321106},{"id":"https://openalex.org/C171268870","wikidata":"https://www.wikidata.org/wiki/Q1486676","display_name":"GRASP","level":2,"score":0.5633985996246338},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.550640881061554},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5378656983375549},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5144019722938538},{"id":"https://openalex.org/C145460709","wikidata":"https://www.wikidata.org/wiki/Q859951","display_name":"Human\u2013robot interaction","level":3,"score":0.47885963320732117},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.43469658493995667},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.4332769513130188},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.4101802706718445},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3605367839336395},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3553996682167053},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1642472743988037},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.12985938787460327},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1017/s0263574723001510","is_oa":false,"landing_page_url":"https://doi.org/10.1017/s0263574723001510","pdf_url":null,"source":{"id":"https://openalex.org/S92163612","display_name":"Robotica","issn_l":"0263-5747","issn":["0263-5747","1469-8668"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Robotica","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.49000000953674316,"id":"https://metadata.un.org/sdg/16"},{"display_name":"Reduced inequalities","score":0.44999998807907104,"id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W1542118511","https://openalex.org/W1999156278","https://openalex.org/W2185175083","https://openalex.org/W2186283982","https://openalex.org/W2523315006","https://openalex.org/W2568262903","https://openalex.org/W2572996265","https://openalex.org/W2606473278","https://openalex.org/W2946086442","https://openalex.org/W2953669419","https://openalex.org/W2998012869","https://openalex.org/W3006154882","https://openalex.org/W3033090297","https://openalex.org/W3099131440","https://openalex.org/W3110939063","https://openalex.org/W3112148202","https://openalex.org/W3174073039","https://openalex.org/W3207311111","https://openalex.org/W4221167790","https://openalex.org/W4242073862","https://openalex.org/W4289126595","https://openalex.org/W4308200603","https://openalex.org/W4312456699","https://openalex.org/W4313431304","https://openalex.org/W4320712909","https://openalex.org/W4387197149","https://openalex.org/W4390188602","https://openalex.org/W6757135208","https://openalex.org/W6843266318","https://openalex.org/W6849900021"],"related_works":["https://openalex.org/W2955859849","https://openalex.org/W4288263119","https://openalex.org/W3202500587","https://openalex.org/W3015724364","https://openalex.org/W2967994095","https://openalex.org/W4285240985","https://openalex.org/W2900126711","https://openalex.org/W4286930972","https://openalex.org/W3202115945","https://openalex.org/W2542958340"],"abstract_inverted_index":{"Abstract":[0],"The":[1],"language-guided":[2],"visual":[3,48,89],"robotic":[4],"grasping":[5,56,95],"task":[6],"focuses":[7],"on":[8,15,142,166],"enabling":[9],"robots":[10,78],"to":[11,116,123,145,170],"grasp":[12],"objects":[13,84,121],"based":[14],"human":[16,99],"language":[17,29,50],"instructions.":[18,100],"However,":[19],"real-world":[20],"human-robot":[21,164],"collaboration":[22],"tasks":[23],"often":[24],"involve":[25],"situations":[26],"with":[27,98,161],"ambiguous":[28],"instructions":[30],"and":[31,49,52,88,134],"complex":[32],"scenarios.":[33],"These":[34],"challenges":[35],"arise":[36],"in":[37,47,73,79,96,124,152],"the":[38,59,125,128,147,172],"understanding":[39],"of":[40,44,54,83,105,149,174],"linguistic":[41],"queries,":[42],"discrimination":[43],"key":[45],"concepts":[46],"information,":[51],"generation":[53],"executable":[55],"configurations":[57],"for":[58,120],"robot\u2019s":[60],"end-effector.":[61],"To":[62],"overcome":[63],"these":[64],"challenges,":[65],"we":[66],"propose":[67],"a":[68,110,167],"novel":[69],"multi-modal":[70,118],"transformer-based":[71],"framework":[72,92,103,129],"this":[74],"study,":[75],"which":[76],"assists":[77],"localizing":[80],"spatial":[81,132],"interactions":[82,119,165],"using":[85],"text":[86],"queries":[87],"sensing.":[90],"This":[91],"facilitates":[93],"object":[94],"accordance":[97],"Our":[101],"developed":[102],"consists":[104],"two":[106],"main":[107],"components.":[108],"First,":[109],"visual-linguistic":[111],"transformer":[112],"encoder":[113],"is":[114],"employed":[115],"model":[117],"referred":[122],"text.":[126],"Second,":[127],"performs":[130],"joint":[131],"localization":[133],"grasping.":[135],"Extensive":[136],"ablation":[137],"studies":[138],"have":[139,158],"been":[140,159],"conducted":[141],"multiple":[143],"datasets":[144],"evaluate":[146],"advantages":[148],"each":[150],"component":[151],"our":[153,175],"model.":[154],"Additionally,":[155],"physical":[156,168],"experiments":[157],"performed":[160],"natural":[162],"language-driven":[163],"robot":[169],"validate":[171],"practicality":[173],"approach.":[176]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":1}],"updated_date":"2026-08-28T12:50:07.497085","created_date":"2025-10-10T00:00:00"}