{"id":"https://openalex.org/W4401414944","doi":"https://doi.org/10.1109/icra57147.2024.10611462","title":"Language and Sketching: An LLM-driven Interactive Multimodal Multitask Robot Navigation Framework","display_name":"Language and Sketching: An LLM-driven Interactive Multimodal Multitask Robot Navigation Framework","publication_year":2024,"publication_date":"2024-05-13","ids":{"openalex":"https://openalex.org/W4401414944","doi":"https://doi.org/10.1109/icra57147.2024.10611462"},"language":"en","primary_location":{"id":"doi:10.1109/icra57147.2024.10611462","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra57147.2024.10611462","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5059080410","display_name":"Weiqin Zu","orcid":"https://orcid.org/0009-0002-9745-4457"},"institutions":[{"id":"https://openalex.org/I30809798","display_name":"ShanghaiTech University","ror":"https://ror.org/030bhh786","country_code":"CN","type":"education","lineage":["https://openalex.org/I30809798"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Weiqin Zu","raw_affiliation_strings":["ShanghaiTech University,Shanghai,China"],"affiliations":[{"raw_affiliation_string":"ShanghaiTech University,Shanghai,China","institution_ids":["https://openalex.org/I30809798"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108913285","display_name":"Wenbin Song","orcid":"https://orcid.org/0000-0002-8399-7189"},"institutions":[{"id":"https://openalex.org/I30809798","display_name":"ShanghaiTech University","ror":"https://ror.org/030bhh786","country_code":"CN","type":"education","lineage":["https://openalex.org/I30809798"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenbin Song","raw_affiliation_strings":["ShanghaiTech University,Shanghai,China"],"affiliations":[{"raw_affiliation_string":"ShanghaiTech University,Shanghai,China","institution_ids":["https://openalex.org/I30809798"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101661883","display_name":"Ruiqing Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I30809798","display_name":"ShanghaiTech University","ror":"https://ror.org/030bhh786","country_code":"CN","type":"education","lineage":["https://openalex.org/I30809798"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruiqing Chen","raw_affiliation_strings":["ShanghaiTech University,Shanghai,China"],"affiliations":[{"raw_affiliation_string":"ShanghaiTech University,Shanghai,China","institution_ids":["https://openalex.org/I30809798"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026734390","display_name":"Ze Guo","orcid":null},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ze Guo","raw_affiliation_strings":["Harbin Institute of Technology,Harbin,China"],"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology,Harbin,China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063874810","display_name":"Fanglei Sun","orcid":"https://orcid.org/0000-0002-4302-2512"},"institutions":[{"id":"https://openalex.org/I148128674","display_name":"University of Shanghai for Science and Technology","ror":"https://ror.org/00ay9v204","country_code":"CN","type":"education","lineage":["https://openalex.org/I148128674"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fanglei Sun","raw_affiliation_strings":["University of Shanghai for Science and Technology,Shanghai,China"],"affiliations":[{"raw_affiliation_string":"University of Shanghai for Science and Technology,Shanghai,China","institution_ids":["https://openalex.org/I148128674"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zheng Tian","orcid":null},"institutions":[{"id":"https://openalex.org/I30809798","display_name":"ShanghaiTech University","ror":"https://ror.org/030bhh786","country_code":"CN","type":"education","lineage":["https://openalex.org/I30809798"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zheng Tian","raw_affiliation_strings":["ShanghaiTech University,Shanghai,China"],"affiliations":[{"raw_affiliation_string":"ShanghaiTech University,Shanghai,China","institution_ids":["https://openalex.org/I30809798"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100721259","display_name":"Wei Pan","orcid":"https://orcid.org/0000-0003-1121-9879"},"institutions":[{"id":"https://openalex.org/I28407311","display_name":"University of Manchester","ror":"https://ror.org/027m9bs27","country_code":"GB","type":"education","lineage":["https://openalex.org/I28407311"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Wei Pan","raw_affiliation_strings":["The University of Manchester,Manchester,United Kingdom"],"affiliations":[{"raw_affiliation_string":"The University of Manchester,Manchester,United Kingdom","institution_ids":["https://openalex.org/I28407311"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021575840","display_name":"Jun Wang","orcid":"https://orcid.org/0000-0001-8128-3580"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jun Wang","raw_affiliation_strings":["University College London,London,United Kingdom"],"affiliations":[{"raw_affiliation_string":"University College London,London,United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5059080410"],"corresponding_institution_ids":["https://openalex.org/I30809798"],"apc_list":null,"apc_paid":null,"fwci":5.5724,"has_fulltext":false,"cited_by_count":23,"citation_normalized_percentile":{"value":0.96943844,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1019","last_page":"1025"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10586","display_name":"Robotic Path Planning Algorithms","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7984523773193359},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.6383734941482544},{"id":"https://openalex.org/keywords/multimodal-interaction","display_name":"Multimodal interaction","score":0.5755637884140015},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.5428053140640259},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4543541669845581},{"id":"https://openalex.org/keywords/mobile-robot","display_name":"Mobile robot","score":0.41093355417251587}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7984523773193359},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.6383734941482544},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.5755637884140015},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5428053140640259},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4543541669845581},{"id":"https://openalex.org/C19966478","wikidata":"https://www.wikidata.org/wiki/Q4810574","display_name":"Mobile robot","level":3,"score":0.41093355417251587}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/icra57147.2024.10611462","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra57147.2024.10611462","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.atira.dk:publications/483b646a-140c-4f28-80d0-bcdfb4dbcb52","is_oa":false,"landing_page_url":"https://research.manchester.ac.uk/en/publications/483b646a-140c-4f28-80d0-bcdfb4dbcb52","pdf_url":null,"source":{"id":"https://openalex.org/S4306400662","display_name":"Research Explorer (The University of Manchester)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I28407311","host_organization_name":"University of Manchester","host_organization_lineage":["https://openalex.org/I28407311"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Zu, W, Song, W, Chen, R, Guo, Z, Sun, F, Tian, Z, Pan, W & Wang, J 2024, Language and Sketching : An LLM-driven Interactive Multimodal Multitask Robot Navigation Framework. in 2024 IEEE International Conference on Robotics and Automation, ICRA 2024. Proceedings - IEEE International Conference on Robotics and Automation, IEEE, pp. 1019-1025, 2024 IEEE International Conference on Robotics and Automation, ICRA 2024, Yokohama, Japan, 13/05/24. https://doi.org/10.1109/ICRA57147.2024.10611462","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":32,"referenced_works":["https://openalex.org/W192919555","https://openalex.org/W1579787924","https://openalex.org/W2043242772","https://openalex.org/W2105686579","https://openalex.org/W2167052694","https://openalex.org/W2419072270","https://openalex.org/W2538619708","https://openalex.org/W2588505004","https://openalex.org/W2984168904","https://openalex.org/W3022845750","https://openalex.org/W3090832832","https://openalex.org/W3091957856","https://openalex.org/W3109097593","https://openalex.org/W3112631946","https://openalex.org/W3172675210","https://openalex.org/W3196525377","https://openalex.org/W4224912544","https://openalex.org/W4226150156","https://openalex.org/W4285047690","https://openalex.org/W4303648971","https://openalex.org/W4312238277","https://openalex.org/W4313952519","https://openalex.org/W4383066231","https://openalex.org/W4383097638","https://openalex.org/W4384264726","https://openalex.org/W4388660746","https://openalex.org/W4389665575","https://openalex.org/W4394828156","https://openalex.org/W6810640255","https://openalex.org/W6846242362","https://openalex.org/W6854555012","https://openalex.org/W6854929498"],"related_works":["https://openalex.org/W2122871747","https://openalex.org/W3114279067","https://openalex.org/W2035104213","https://openalex.org/W2240967841","https://openalex.org/W2791686160","https://openalex.org/W1587348481","https://openalex.org/W1513257435","https://openalex.org/W2289840258","https://openalex.org/W2503352526","https://openalex.org/W2968405290"],"abstract_inverted_index":{"The":[0],"socially-aware":[1],"navigation":[2,72,84,104],"system":[3],"has":[4,164],"evolved":[5],"to":[6,36,76,116],"adeptly":[7],"avoid":[8],"various":[9],"obstacles":[10],"while":[11],"performing":[12],"multiple":[13,118],"tasks,":[14],"such":[15],"as":[16,103],"point-to-point":[17],"navigation,":[18],"human-following,":[19],"and":[20,54,98,106,137,139,157],"-guiding.":[21],"However,":[22],"a":[23,92,110],"prominent":[24],"gap":[25],"persists:":[26],"in":[27,82,146,154],"Human-Robot":[28],"Interaction":[29],"(HRI),":[30],"the":[31,43,51,78,83,121,130,142,147,158],"procedure":[32],"of":[33,132,141],"communicating":[34],"commands":[35],"robots":[37],"demands":[38],"intricate":[39],"mathematical":[40],"formulations.":[41],"Furthermore,":[42],"transition":[44],"between":[45],"tasks":[46,119],"does":[47],"not":[48],"quite":[49],"possess":[50],"intuitive":[52],"control":[53,107],"user-centric":[55],"interactivity":[56],"that":[57,162],"one":[58],"would":[59],"desire.":[60],"In":[61],"this":[62,88],"work,":[63],"we":[64],"propose":[65],"an":[66,170],"LLM-driven":[67],"interactive":[68,172],"multimodal":[69,93,133],"multitask":[70,135],"robot":[71],"framework,":[73],"termed":[74],"LIM2N,":[75],"solve":[77],"above":[79],"new":[80],"challenge":[81],"field.":[85],"We":[86],"achieve":[87],"by":[89],"first":[90],"introducing":[91],"interaction":[94],"framework":[95],"where":[96],"language":[97],"hand-drawn":[99],"inputs":[100],"can":[101],"serve":[102],"constraints":[105],"objectives.":[108],"Next,":[109],"reinforcement":[111],"learning":[112],"agent":[113],"is":[114],"built":[115],"handle":[117],"with":[120],"received":[122],"information.":[123],"Crucially,":[124],"LIM2N":[125,163],"creates":[126],"smooth":[127],"cooperation":[128],"among":[129],"reasoning":[131],"input,":[134],"planning,":[136],"adaptation":[138],"processing":[140],"intelligent":[143],"sensing":[144],"modules":[145],"complicated":[148],"system.":[149],"Detailed":[150],"experiments":[151],"are":[152],"conducted":[153],"both":[155],"simulation":[156],"real":[159],"world":[160],"demonstrating":[161],"solid":[165],"user":[166],"needs":[167],"understanding,":[168],"alongside":[169],"enhanced":[171],"experience.":[173]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":20},{"year":2024,"cited_by_count":2}],"updated_date":"2026-04-16T08:26:57.006410","created_date":"2025-10-10T00:00:00"}