{"id":"https://openalex.org/W7134848214","doi":"https://doi.org/10.1109/lra.2026.3671701","title":"Simultaneous Tactile-Visual Perception for Learning Multimodal Robot Manipulation","display_name":"Simultaneous Tactile-Visual Perception for Learning Multimodal Robot Manipulation","publication_year":2026,"publication_date":"2026-03-09","ids":{"openalex":"https://openalex.org/W7134848214","doi":"https://doi.org/10.1109/lra.2026.3671701"},"language":null,"primary_location":{"id":"doi:10.1109/lra.2026.3671701","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lra.2026.3671701","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766","2377-3774"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100317619","display_name":"Yuyang Li","orcid":"https://orcid.org/0000-0002-5794-7997"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuyang Li","raw_affiliation_strings":["Institute for Artificial Intelligence, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-5794-7997","affiliations":[{"raw_affiliation_string":"Institute for Artificial Intelligence, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570","https://openalex.org/I4210100255"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yinghan Chen","orcid":"https://orcid.org/0009-0002-7110-8097"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yinghan Chen","raw_affiliation_strings":["Institute for Artificial Intelligence, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0002-7110-8097","affiliations":[{"raw_affiliation_string":"Institute for Artificial Intelligence, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570","https://openalex.org/I4210100255"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101399290","display_name":"Zihang Zhao","orcid":"https://orcid.org/0000-0003-3215-7152"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zihang Zhao","raw_affiliation_strings":["Institute for Artificial Intelligence, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3215-7152","affiliations":[{"raw_affiliation_string":"Institute for Artificial Intelligence, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570","https://openalex.org/I4210100255"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023051708","display_name":"Puhao Li","orcid":"https://orcid.org/0009-0003-2696-9346"},"institutions":[{"id":"https://openalex.org/I4210126590","display_name":"General Research Institute for Nonferrous Metals (China)","ror":"https://ror.org/03d8em588","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210126590"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Puhao Li","raw_affiliation_strings":["Beijing Institute for General AI, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0003-2696-9346","affiliations":[{"raw_affiliation_string":"Beijing Institute for General AI, Beijing, China","institution_ids":["https://openalex.org/I4210126590"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Tengyu Liu","orcid":"https://orcid.org/0000-0003-4006-1740"},"institutions":[{"id":"https://openalex.org/I4210126590","display_name":"General Research Institute for Nonferrous Metals (China)","ror":"https://ror.org/03d8em588","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210126590"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tengyu Liu","raw_affiliation_strings":["Beijing Institute for General AI, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-4006-1740","affiliations":[{"raw_affiliation_string":"Beijing Institute for General AI, Beijing, China","institution_ids":["https://openalex.org/I4210126590"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103357399","display_name":"S. H. Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210126590","display_name":"General Research Institute for Nonferrous Metals (China)","ror":"https://ror.org/03d8em588","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210126590"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Siyuan Huang","raw_affiliation_strings":["Beijing Institute for General AI, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1524-7148","affiliations":[{"raw_affiliation_string":"Beijing Institute for General AI, Beijing, China","institution_ids":["https://openalex.org/I4210126590"]}]},{"author_position":"last","author":{"id":null,"display_name":"Yixin Zhu","orcid":"https://orcid.org/0000-0001-7024-1545"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yixin Zhu","raw_affiliation_strings":["Institute for Artificial Intelligence, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-7024-1545","affiliations":[{"raw_affiliation_string":"Institute for Artificial Intelligence, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570","https://openalex.org/I4210100255"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.9653,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.93882192,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"11","issue":"4","first_page":"5254","last_page":"5261"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.39959999918937683,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.39959999918937683,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10338","display_name":"Advanced Sensor and Energy Harvesting Materials","score":0.3921000063419342,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10914","display_name":"Tactile and Sensory Interactions","score":0.04450000077486038,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.5770999789237976},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"Multimodal learning","score":0.510699987411499},{"id":"https://openalex.org/keywords/imitation","display_name":"Imitation","score":0.4991999864578247},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.4805000126361847},{"id":"https://openalex.org/keywords/tactile-perception","display_name":"Tactile perception","score":0.45559999346733093},{"id":"https://openalex.org/keywords/tactile-sensor","display_name":"Tactile sensor","score":0.39100000262260437},{"id":"https://openalex.org/keywords/active-perception","display_name":"Active perception","score":0.3869999945163727},{"id":"https://openalex.org/keywords/wired-glove","display_name":"Wired glove","score":0.36340001225471497}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6959999799728394},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6103000044822693},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.5770999789237976},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5401999950408936},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.510699987411499},{"id":"https://openalex.org/C126388530","wikidata":"https://www.wikidata.org/wiki/Q1131737","display_name":"Imitation","level":2,"score":0.4991999864578247},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.4805000126361847},{"id":"https://openalex.org/C3017819093","wikidata":"https://www.wikidata.org/wiki/Q328835","display_name":"Tactile perception","level":3,"score":0.45559999346733093},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4018999934196472},{"id":"https://openalex.org/C46722567","wikidata":"https://www.wikidata.org/wiki/Q7674139","display_name":"Tactile sensor","level":3,"score":0.39100000262260437},{"id":"https://openalex.org/C2776010242","wikidata":"https://www.wikidata.org/wiki/Q4677575","display_name":"Active perception","level":3,"score":0.3869999945163727},{"id":"https://openalex.org/C107953548","wikidata":"https://www.wikidata.org/wiki/Q1172433","display_name":"Wired glove","level":3,"score":0.36340001225471497},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.3589000105857849},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.3529999852180481},{"id":"https://openalex.org/C188888258","wikidata":"https://www.wikidata.org/wiki/Q7353390","display_name":"Robot learning","level":4,"score":0.3513999879360199},{"id":"https://openalex.org/C60692881","wikidata":"https://www.wikidata.org/wiki/Q584529","display_name":"Humanoid robot","level":3,"score":0.3337000012397766},{"id":"https://openalex.org/C178253425","wikidata":"https://www.wikidata.org/wiki/Q162668","display_name":"Visual perception","level":3,"score":0.32330000400543213},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.32120001316070557},{"id":"https://openalex.org/C22033958","wikidata":"https://www.wikidata.org/wiki/Q7167036","display_name":"Perceptual learning","level":3,"score":0.2994999885559082},{"id":"https://openalex.org/C145460709","wikidata":"https://www.wikidata.org/wiki/Q859951","display_name":"Human\u2013robot interaction","level":3,"score":0.2994999885559082},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.2854999899864197},{"id":"https://openalex.org/C2779321571","wikidata":"https://www.wikidata.org/wiki/Q7936605","display_name":"Visual learning","level":2,"score":0.2533000111579895},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.2500999867916107}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/lra.2026.3671701","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lra.2026.3671701","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766","2377-3774"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G25600548","display_name":null,"funder_award_id":"62376009","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":49,"referenced_works":["https://openalex.org/W2105934661","https://openalex.org/W2128681845","https://openalex.org/W2153994943","https://openalex.org/W2410441722","https://openalex.org/W2569244399","https://openalex.org/W2775635818","https://openalex.org/W2781493652","https://openalex.org/W2783627282","https://openalex.org/W2799692011","https://openalex.org/W2922176345","https://openalex.org/W2950069298","https://openalex.org/W2966952767","https://openalex.org/W2967428101","https://openalex.org/W3007580291","https://openalex.org/W3008303301","https://openalex.org/W3040477237","https://openalex.org/W3138748647","https://openalex.org/W3190214329","https://openalex.org/W4221056281","https://openalex.org/W4281900509","https://openalex.org/W4285102140","https://openalex.org/W4285218426","https://openalex.org/W4285741815","https://openalex.org/W4312865837","https://openalex.org/W4376605745","https://openalex.org/W4385245566","https://openalex.org/W4385430674","https://openalex.org/W4385756471","https://openalex.org/W4385975773","https://openalex.org/W4386473175","https://openalex.org/W4389352581","https://openalex.org/W4390776907","https://openalex.org/W4401415473","https://openalex.org/W4402354127","https://openalex.org/W4403337227","https://openalex.org/W4403863408","https://openalex.org/W4403941282","https://openalex.org/W4404307277","https://openalex.org/W4404851033","https://openalex.org/W4405753615","https://openalex.org/W4405785291","https://openalex.org/W4409326585","https://openalex.org/W4410115318","https://openalex.org/W4411157391","https://openalex.org/W4412722087","https://openalex.org/W4413925959","https://openalex.org/W4413926146","https://openalex.org/W4414199183","https://openalex.org/W7123872314"],"related_works":[],"abstract_inverted_index":{"Robotic":[0],"manipulation":[1,62,160],"requires":[2],"both":[3],"rich":[4,57],"multimodal":[5,46,58,87,162,170],"perception":[6,47,75,171],"and":[7,22,48,103,140,155,158],"effective":[8],"learning":[9,32,82,108,174],"frameworks":[10,175],"to":[11],"handle":[12],"complex":[13],"real-world":[14,122],"tasks.":[15],"See-Through-Skin":[16,41,71],"(STS)":[17,42,72],"sensors,":[18],"which":[19],"combine":[20],"tactile":[21,52,137],"visual":[23],"perception,":[24],"offer":[25],"promising":[26],"sensing":[27],"capabilities,":[28],"while":[29,106],"modern":[30,173],"imitation":[31,81],"provides":[33],"powerful":[34],"tools":[35],"for":[36,89],"policy":[37,138,142],"acquisition.":[38],"However,":[39],"existing":[40],"designs":[43],"lack":[44],"simultaneous":[45,169],"suffer":[49],"from":[50],"unreliable":[51],"tracking.":[53],"Furthermore,":[54],"integrating":[55],"these":[56,86,111],"signals":[59,88,112],"into":[60],"learning-based":[61],"pipelines":[63],"remains":[64],"an":[65,70,80,126],"open":[66],"challenge.":[67],"We":[68],"introduceTacThru,":[69],"sensor":[73,92],"enablingsimultaneousvisual":[74],"androbusttactile":[76],"signal":[77],"extraction,":[78],"andTacThru-UMI,":[79],"framework":[83],"that":[84,167],"leverages":[85],"manipulation.":[90,181],"Our":[91],"features":[93],"a":[94,114],"fully":[95],"transparent":[96],"elastomer,":[97],"persistent":[98],"illumination,":[99],"novel":[100],"keyline":[101],"markers,":[102],"efficient":[104],"tracking,":[105],"our":[107],"system":[109,145],"integrates":[110],"through":[113],"Transformer-based":[115],"Diffusion":[116],"Policy.":[117],"Experiments":[118],"on":[119],"five":[120],"challenging":[121],"tasks":[123],"show":[124],"thatTacThru-UMIachieves":[125],"average":[127],"success":[128],"rate":[129],"of":[130,136],"85.5%,":[131],"significantly":[132],"outperforming":[133],"the":[134],"baselines":[135],"(66.3%)":[139],"vision-only":[141],"(55.4%).":[143],"The":[144],"excels":[146],"in":[147],"critical":[148],"scenarios,":[149],"including":[150],"contact":[151],"detection":[152],"with":[153,172],"thin":[154],"soft":[156],"objects":[157],"precision":[159],"requiring":[161],"coordination.":[163],"This":[164],"work":[165],"demonstrates":[166],"combining":[168],"enables":[176],"more":[177],"precise,":[178],"adaptable":[179],"robotic":[180]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2026-02-12T00:00:00"}