{"id":"https://openalex.org/W4390776907","doi":"https://doi.org/10.1109/tro.2024.3352969","title":"General-Purpose Sim2Real Protocol for Learning Contact-Rich Manipulation With Marker-Based Visuotactile Sensors","display_name":"General-Purpose Sim2Real Protocol for Learning Contact-Rich Manipulation With Marker-Based Visuotactile Sensors","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4390776907","doi":"https://doi.org/10.1109/tro.2024.3352969"},"language":"en","primary_location":{"id":"doi:10.1109/tro.2024.3352969","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tro.2024.3352969","pdf_url":"https://ieeexplore.ieee.org/ielx7/8860/4359257/10388459.pdf","source":{"id":"https://openalex.org/S144620930","display_name":"IEEE Transactions on Robotics","issn_l":"1552-3098","issn":["1552-3098","1546-1904","1941-0468"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Robotics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://ieeexplore.ieee.org/ielx7/8860/4359257/10388459.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088873598","display_name":"Weihang Chen","orcid":"https://orcid.org/0000-0003-1892-5530"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weihang Chen","raw_affiliation_strings":["Department of Mechanical Engineering, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1892-5530","affiliations":[{"raw_affiliation_string":"Department of Mechanical Engineering, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087396510","display_name":"Jing Xu","orcid":"https://orcid.org/0000-0002-2997-2351"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jing Xu","raw_affiliation_strings":["Department of Mechanical Engineering, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2997-2351","affiliations":[{"raw_affiliation_string":"Department of Mechanical Engineering, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008903324","display_name":"Fanbo Xiang","orcid":"https://orcid.org/0009-0005-5335-873X"},"institutions":[{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fanbo Xiang","raw_affiliation_strings":["Department of Computer Science and Engineering, University of California, San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0005-5335-873X","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University of California, San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060983535","display_name":"Xiaodi Yuan","orcid":"https://orcid.org/0009-0003-2320-2297"},"institutions":[{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaodi Yuan","raw_affiliation_strings":["Department of Computer Science and Engineering, University of California, San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0003-2320-2297","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University of California, San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056245495","display_name":"Hao Su","orcid":"https://orcid.org/0000-0002-1796-2682"},"institutions":[{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hao Su","raw_affiliation_strings":["Department of Computer Science and Engineering, University of California, San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-1796-2682","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University of California, San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100403640","display_name":"Rui Chen","orcid":"https://orcid.org/0000-0002-4041-4131"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rui Chen","raw_affiliation_strings":["Department of Mechanical Engineering, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-4041-4131","affiliations":[{"raw_affiliation_string":"Department of Mechanical Engineering, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.8911,"has_fulltext":true,"cited_by_count":18,"citation_normalized_percentile":{"value":0.95655289,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"40","issue":null,"first_page":"1509","last_page":"1526"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10914","display_name":"Tactile and Sensory Interactions","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10914","display_name":"Tactile and Sensory Interactions","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10338","display_name":"Advanced Sensor and Energy Harvesting Materials","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"Generalizability theory","score":0.7726397514343262},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6409306526184082},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6047937870025635},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.576677680015564},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.5595279335975647},{"id":"https://openalex.org/keywords/tactile-sensor","display_name":"Tactile sensor","score":0.43939894437789917},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.43874409794807434},{"id":"https://openalex.org/keywords/pixel","display_name":"Pixel","score":0.42236852645874023},{"id":"https://openalex.org/keywords/contact-force","display_name":"Contact force","score":0.4156697690486908},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3480907082557678},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.33352887630462646},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.15828433632850647},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08477884531021118}],"concepts":[{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.7726397514343262},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6409306526184082},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6047937870025635},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.576677680015564},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.5595279335975647},{"id":"https://openalex.org/C46722567","wikidata":"https://www.wikidata.org/wiki/Q7674139","display_name":"Tactile sensor","level":3,"score":0.43939894437789917},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.43874409794807434},{"id":"https://openalex.org/C160633673","wikidata":"https://www.wikidata.org/wiki/Q355198","display_name":"Pixel","level":2,"score":0.42236852645874023},{"id":"https://openalex.org/C81302111","wikidata":"https://www.wikidata.org/wiki/Q2916417","display_name":"Contact force","level":2,"score":0.4156697690486908},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3480907082557678},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33352887630462646},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.15828433632850647},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08477884531021118},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C204787440","wikidata":"https://www.wikidata.org/wiki/Q188504","display_name":"Alternative medicine","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tro.2024.3352969","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tro.2024.3352969","pdf_url":"https://ieeexplore.ieee.org/ielx7/8860/4359257/10388459.pdf","source":{"id":"https://openalex.org/S144620930","display_name":"IEEE Transactions on Robotics","issn_l":"1552-3098","issn":["1552-3098","1546-1904","1941-0468"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Robotics","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1109/tro.2024.3352969","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tro.2024.3352969","pdf_url":"https://ieeexplore.ieee.org/ielx7/8860/4359257/10388459.pdf","source":{"id":"https://openalex.org/S144620930","display_name":"IEEE Transactions on Robotics","issn_l":"1552-3098","issn":["1552-3098","1546-1904","1941-0468"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Robotics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4390776907.pdf","grobid_xml":"https://content.openalex.org/works/W4390776907.grobid-xml"},"referenced_works_count":54,"referenced_works":["https://openalex.org/W2128681845","https://openalex.org/W2142736275","https://openalex.org/W2146990154","https://openalex.org/W2767050701","https://openalex.org/W2775635818","https://openalex.org/W2781493652","https://openalex.org/W2911087563","https://openalex.org/W2917559944","https://openalex.org/W2964198579","https://openalex.org/W2969996603","https://openalex.org/W2972319721","https://openalex.org/W2981030070","https://openalex.org/W3006398608","https://openalex.org/W3029901419","https://openalex.org/W3035624836","https://openalex.org/W3041148233","https://openalex.org/W3048889784","https://openalex.org/W3088310808","https://openalex.org/W3090143443","https://openalex.org/W3101442004","https://openalex.org/W3112669067","https://openalex.org/W3120778962","https://openalex.org/W3126321819","https://openalex.org/W3128582681","https://openalex.org/W3130633756","https://openalex.org/W3173438814","https://openalex.org/W3174011998","https://openalex.org/W3176646219","https://openalex.org/W3192021195","https://openalex.org/W3198417399","https://openalex.org/W3205950068","https://openalex.org/W3206949798","https://openalex.org/W3207777114","https://openalex.org/W3216705422","https://openalex.org/W3216772467","https://openalex.org/W4221163903","https://openalex.org/W4225566379","https://openalex.org/W4226167593","https://openalex.org/W4285232917","https://openalex.org/W4286611167","https://openalex.org/W4292231110","https://openalex.org/W4311057649","https://openalex.org/W4319999962","https://openalex.org/W4322707307","https://openalex.org/W4327522806","https://openalex.org/W4366492958","https://openalex.org/W4389164224","https://openalex.org/W6763422710","https://openalex.org/W6802188089","https://openalex.org/W6804601995","https://openalex.org/W6810080435","https://openalex.org/W6810370022","https://openalex.org/W6810655313","https://openalex.org/W6845226490"],"related_works":["https://openalex.org/W1960616769","https://openalex.org/W3128324021","https://openalex.org/W4289656195","https://openalex.org/W2124146082","https://openalex.org/W2121836998","https://openalex.org/W2709287450","https://openalex.org/W2040114039","https://openalex.org/W2027665490","https://openalex.org/W2106712047","https://openalex.org/W2157335472"],"abstract_inverted_index":{"Visuotactile":[0],"sensors":[1,43],"can":[2,103],"provide":[3],"rich":[4],"contact":[5],"information,":[6],"having":[7],"great":[8],"potential":[9],"in":[10],"contact-rich":[11],"manipulation":[12,39,84],"tasks":[13,40,173],"with":[14,41,87],"reinforcement":[15],"learning":[16,86,188],"(RL)":[17],"policies.":[18,148],"Sim2Real":[19,36,81,152],"technique":[20],"tackles":[21],"the":[22,53,64,71,93,105,126,142,155,160,183,186],"challenge":[23],"of":[24,31,66,128,131,146,162],"RL's":[25],"reliance":[26],"on":[27,45,154,171],"a":[28,79,117,136],"large":[29],"amount":[30],"interaction":[32],"data.":[33],"However,":[34],"most":[35],"methods":[37,60],"for":[38,69,83,111,193],"visuotactile":[42,89],"rely":[44],"rigid-body":[46],"physics":[47,100],"simulation,":[48],"which":[49],"fails":[50],"to":[51,140,158],"simulate":[52,104],"real":[54],"elastic":[55],"deformation":[56,107],"precisely.":[57],"Moreover,":[58],"these":[59],"do":[61],"not":[62],"exploit":[63],"characteristic":[65],"tactile":[67,119,132],"signals":[68],"designing":[70],"network":[72,122],"architecture.":[73],"In":[74],"this":[75],"paper,":[76],"we":[77,96,166],"build":[78],"general-purpose":[80],"protocol":[82],"policy":[85,187],"marker-based":[88],"sensors.":[90],"To":[91],"improve":[92,141],"simulation":[94],"fidelity,":[95],"employ":[97],"an":[98],"FEM-based":[99],"simulator":[101,184],"that":[102,123],"sensor":[106,133],"accurately":[108],"and":[109,135,144,177,185],"stably":[110],"arbitrary":[112],"geometries.":[113],"We":[114,149],"further":[115,167],"propose":[116],"novel":[118],"feature":[120],"extraction":[121],"directly":[124],"processes":[125],"set":[127],"pixel":[129],"coordinates":[130],"markers":[134],"self-supervised":[137],"pre-training":[138],"strategy":[139],"efficiency":[143],"generalizability":[145,170],"RL":[147],"conduct":[150],"extensive":[151],"experiments":[153],"peg-in-hole":[156],"task":[157],"validate":[159],"effectiveness":[161],"our":[163],"method.":[164],"And":[165],"show":[168],"its":[169],"additional":[172],"including":[174,182],"plug":[175],"adjustment":[176],"lock":[178],"opening.":[179],"The":[180],"protocol,":[181],"framework,":[189],"will":[190],"be":[191],"open-sourced":[192],"community":[194],"usage.":[195]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":3}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}