{"id":"https://openalex.org/W7135180683","doi":"https://doi.org/10.1109/lra.2026.3674003","title":"Language-Driven Multi-Task Manipulation With Action-Mask-Enhanced Multimodal Learning","display_name":"Language-Driven Multi-Task Manipulation With Action-Mask-Enhanced Multimodal Learning","publication_year":2026,"publication_date":"2026-03-13","ids":{"openalex":"https://openalex.org/W7135180683","doi":"https://doi.org/10.1109/lra.2026.3674003"},"language":null,"primary_location":{"id":"doi:10.1109/lra.2026.3674003","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lra.2026.3674003","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766","2377-3774"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075703600","display_name":"Ningquan Gu","orcid":"https://orcid.org/0000-0002-7032-900X"},"institutions":[{"id":"https://openalex.org/I201537933","display_name":"Tohoku University","ror":"https://ror.org/01dq60k83","country_code":"JP","type":"education","lineage":["https://openalex.org/I201537933"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Ningquan Gu","raw_affiliation_strings":["Neuro-Robotics Laboratory, Department of Robotics, Graduate School of Engineering, Tohoku University, Sendai, Japan"],"raw_orcid":"https://orcid.org/0000-0002-7032-900X","affiliations":[{"raw_affiliation_string":"Neuro-Robotics Laboratory, Department of Robotics, Graduate School of Engineering, Tohoku University, Sendai, Japan","institution_ids":["https://openalex.org/I201537933"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128972266","display_name":"Yuquan Li","orcid":null},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuquan Li","raw_affiliation_strings":["School of Artificial Intelligence and Automation, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":"https://orcid.org/0009-0001-7676-2656","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence and Automation, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129056972","display_name":"Kazuhiro Kosuge","orcid":null},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Kazuhiro Kosuge","raw_affiliation_strings":["Department of Electrical and Electronic Engineering, Faculty of Engineering, The University of Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0002-2060-2936","affiliations":[{"raw_affiliation_string":"Department of Electrical and Electronic Engineering, Faculty of Engineering, The University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5128949612","display_name":"Mitsuhiro Hayashibe","orcid":null},"institutions":[{"id":"https://openalex.org/I201537933","display_name":"Tohoku University","ror":"https://ror.org/01dq60k83","country_code":"JP","type":"education","lineage":["https://openalex.org/I201537933"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Mitsuhiro Hayashibe","raw_affiliation_strings":["Neuro-Robotics Laboratory, Department of Robotics, Graduate School of Engineering, Tohoku University, Sendai, Japan"],"raw_orcid":"https://orcid.org/0000-0001-6179-5706","affiliations":[{"raw_affiliation_string":"Neuro-Robotics Laboratory, Department of Robotics, Graduate School of Engineering, Tohoku University, Sendai, Japan","institution_ids":["https://openalex.org/I201537933"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.26591786,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"11","issue":"5","first_page":"5550","last_page":"5557"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.2045000046491623,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.2045000046491623,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.16509999334812164,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.1476999968290329,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.3244999945163727},{"id":"https://openalex.org/keywords/multimodality","display_name":"Multimodality","score":0.30390000343322754},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.29809999465942383},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.26339998841285706},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.26089999079704285}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5807999968528748},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42570000886917114},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.39640000462532043},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3244999945163727},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.30390000343322754},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.29809999465942383},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.26339998841285706},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.26089999079704285},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.2468000054359436},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.24079999327659607}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/lra.2026.3674003","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lra.2026.3674003","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766","2377-3774"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W2135639338","https://openalex.org/W2560647685","https://openalex.org/W4385430674","https://openalex.org/W4401415710","https://openalex.org/W4402354098","https://openalex.org/W4403337227","https://openalex.org/W4404781148","https://openalex.org/W4409363400","https://openalex.org/W4412079960","https://openalex.org/W4413147045"],"related_works":[],"abstract_inverted_index":{"For":[0],"long-horizon":[1],"multi-task":[2],"robotic":[3,197],"manipulation,":[4],"hierarchical":[5],"approaches":[6],"provide":[7],"an":[8,41,97,116,137],"effective":[9],"way":[10],"to":[11,78],"combine":[12],"high-level":[13],"language-based":[14],"task":[15,32,73,82],"planning":[16],"with":[17,34,71,102,119,172],"low-level":[18,37],"vision-language":[19],"based":[20],"sub-task":[21,150],"execution.":[22],"Then,":[23],"we":[24],"propose":[25],"a":[26,30,35,48,61,72,80,194,200],"framework":[27],"that":[28,141],"integrates":[29],"two-stage":[31,85],"planner":[33,39],"multimodal":[36,99],"action":[38],"incorporating":[40],"explicit":[42,138],"action-mask":[43,139],"policy.":[44],"At":[45,94],"high":[46],"level,":[47,96],"Vision-Language":[49],"Model":[50,64],"(VLM)":[51],"first":[52],"perceives":[53],"object":[54],"and":[55,60,75,92,109,128,145,153,166,190],"scene":[56],"information":[57],"from":[58],"observations,":[59],"Large":[62],"Language":[63],"(LLM)":[65],"then":[66],"reasons":[67],"over":[68],"this":[69],"together":[70],"library":[74],"human":[76],"instruction":[77],"generate":[79],"textual":[81],"plan.":[83],"This":[84],"design":[86],"mitigates":[87],"modality":[88],"bias":[89],"between":[90],"perception":[91],"planning.":[93],"low":[95],"asymmetric":[98],"encoder,":[100],"SigLIP2":[101],"Weight-Decomposed":[103],"Low-Rank":[104],"Adaptation":[105],"(DoRA)":[106],"for":[107,112,131],"text":[108],"multi-view":[110],"ResNets":[111],"vision,":[113],"feeds":[114],"into":[115],"Action":[117],"Chunking":[118],"Transformer":[120],"(ACT)-based":[121],"policy":[122,140],"enhanced":[123],"by":[124],"Temperature-Scaled":[125],"Spatial":[126],"Attention":[127],"Bidirectional":[129],"Cross-Attention":[130],"language-vision":[132],"fusion.":[133],"We":[134],"further":[135],"introduce":[136],"jointly":[142],"predicts":[143],"actions":[144],"their":[146],"validity,":[147],"enabling":[148],"real-time":[149],"termination":[151],"detection":[152],"robust":[154],"switching":[155],"across":[156],"multiple":[157],"sub-tasks":[158],"without":[159],"additional":[160],"inference":[161],"overhead.":[162],"Experiments":[163],"on":[164,193],"weighing":[165],"multi-object":[167],"manipulation":[168],"tasks":[169],"demonstrate":[170],"performance,":[171],"ablation":[173],"studies":[174],"validating":[175],"the":[176,188],"contribution":[177],"of":[178],"each":[179],"component.":[180],"Generalization":[181],"experiments":[182],"under":[183],"out-of-distribution":[184],"conditions":[185],"highlight":[186],"both":[187],"strengths":[189],"limitations.":[191],"Deployment":[192],"distinct":[195],"dual-arm":[196],"platform":[198],"in":[199],"new":[201],"scenario":[202],"validates":[203],"transferability.":[204]},"counts_by_year":[],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2026-03-14T00:00:00"}