{"id":"https://openalex.org/W2897170587","doi":"https://doi.org/10.1109/icra.2019.8793485","title":"Making Sense of Vision and Touch: Self-Supervised Learning of Multimodal Representations for Contact-Rich Tasks","display_name":"Making Sense of Vision and Touch: Self-Supervised Learning of Multimodal Representations for Contact-Rich Tasks","publication_year":2019,"publication_date":"2019-05-01","ids":{"openalex":"https://openalex.org/W2897170587","doi":"https://doi.org/10.1109/icra.2019.8793485","mag":"2897170587"},"language":"en","primary_location":{"id":"doi:10.1109/icra.2019.8793485","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra.2019.8793485","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1810.10191","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009589018","display_name":"Michelle A. Lee","orcid":"https://orcid.org/0000-0002-9893-3591"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Michelle A. Lee","raw_affiliation_strings":["Department of Computer Science, Stanford University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030826237","display_name":"Yuke Zhu","orcid":"https://orcid.org/0000-0002-9198-2227"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuke Zhu","raw_affiliation_strings":["Department of Computer Science, Stanford University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101973028","display_name":"Krishnan Srinivasan","orcid":"https://orcid.org/0000-0002-0692-1332"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Krishnan Srinivasan","raw_affiliation_strings":["Department of Computer Science, Stanford University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043407770","display_name":"Parth Shah","orcid":"https://orcid.org/0000-0003-0780-0847"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Parth Shah","raw_affiliation_strings":["Department of Computer Science, Stanford University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042646536","display_name":"Silvio Savarese","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Silvio Savarese","raw_affiliation_strings":["Department of Computer Science, Stanford University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100450462","display_name":"Li Fei-Fei","orcid":"https://orcid.org/0000-0002-7481-0810"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li Fei-Fei","raw_affiliation_strings":["Department of Computer Science, Stanford University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061193324","display_name":"Animesh Garg","orcid":"https://orcid.org/0000-0003-0482-4296"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Animesh Garg","raw_affiliation_strings":["Department of Computer Science, Stanford University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021676288","display_name":"Jeannette Bohg","orcid":"https://orcid.org/0000-0002-4921-7193"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeannette Bohg","raw_affiliation_strings":["Department of Computer Science, Stanford University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Stanford University","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5009589018"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":4.3682,"has_fulltext":true,"cited_by_count":32,"citation_normalized_percentile":{"value":0.94620867,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"8943","last_page":"8950"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":1.0,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":1.0,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10914","display_name":"Tactile and Sensory Interactions","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7453930974006653},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7388582825660706},{"id":"https://openalex.org/keywords/haptic-technology","display_name":"Haptic technology","score":0.7252402305603027},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6841943860054016},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.628049910068512},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.6176578998565674},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.5872745513916016},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.5272887349128723},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4740293323993683},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.44941577315330505},{"id":"https://openalex.org/keywords/controller","display_name":"Controller (irrigation)","score":0.4418712854385376},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3443698287010193},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09455734491348267}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7453930974006653},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7388582825660706},{"id":"https://openalex.org/C152086174","wikidata":"https://www.wikidata.org/wiki/Q3030571","display_name":"Haptic technology","level":2,"score":0.7252402305603027},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6841943860054016},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.628049910068512},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.6176578998565674},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5872745513916016},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5272887349128723},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4740293323993683},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.44941577315330505},{"id":"https://openalex.org/C203479927","wikidata":"https://www.wikidata.org/wiki/Q5165939","display_name":"Controller (irrigation)","level":2,"score":0.4418712854385376},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3443698287010193},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09455734491348267},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C6557445","wikidata":"https://www.wikidata.org/wiki/Q173113","display_name":"Agronomy","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/icra.2019.8793485","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra.2019.8793485","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1810.10191","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1810.10191","pdf_url":"https://arxiv.org/pdf/1810.10191","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2897170587","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1810.10191.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1810.10191","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1810.10191","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1810.10191","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1810.10191","pdf_url":"https://arxiv.org/pdf/1810.10191","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320315934","display_name":"Toyota Research Institute","ror":null}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2897170587.pdf"},"referenced_works_count":75,"referenced_works":["https://openalex.org/W779291494","https://openalex.org/W1771410628","https://openalex.org/W1778115449","https://openalex.org/W1932061758","https://openalex.org/W1966487689","https://openalex.org/W1969583232","https://openalex.org/W2008731016","https://openalex.org/W2047887570","https://openalex.org/W2057299504","https://openalex.org/W2066379939","https://openalex.org/W2070338807","https://openalex.org/W2082971343","https://openalex.org/W2100235553","https://openalex.org/W2101340954","https://openalex.org/W2102727145","https://openalex.org/W2110698781","https://openalex.org/W2156102255","https://openalex.org/W2160499119","https://openalex.org/W2162000109","https://openalex.org/W2164587673","https://openalex.org/W2173248099","https://openalex.org/W2184188583","https://openalex.org/W2199073508","https://openalex.org/W2210408922","https://openalex.org/W2221877149","https://openalex.org/W2296055201","https://openalex.org/W2337977475","https://openalex.org/W2345626358","https://openalex.org/W2474174344","https://openalex.org/W2528489519","https://openalex.org/W2529601334","https://openalex.org/W2537677623","https://openalex.org/W2567455162","https://openalex.org/W2567729074","https://openalex.org/W2613677041","https://openalex.org/W2619543829","https://openalex.org/W2765363933","https://openalex.org/W2765726929","https://openalex.org/W2766398089","https://openalex.org/W2767050701","https://openalex.org/W2785962646","https://openalex.org/W2787666871","https://openalex.org/W2790924949","https://openalex.org/W2804941773","https://openalex.org/W2885163910","https://openalex.org/W2919115771","https://openalex.org/W2949382160","https://openalex.org/W2949608212","https://openalex.org/W2962679776","https://openalex.org/W2962899390","https://openalex.org/W2962960500","https://openalex.org/W2963021155","https://openalex.org/W2963280855","https://openalex.org/W2963654998","https://openalex.org/W2963713397","https://openalex.org/W2964112890","https://openalex.org/W2964161785","https://openalex.org/W3000962825","https://openalex.org/W3022259169","https://openalex.org/W3101442004","https://openalex.org/W3105287169","https://openalex.org/W6606244218","https://openalex.org/W6638018090","https://openalex.org/W6638134809","https://openalex.org/W6677477928","https://openalex.org/W6682849425","https://openalex.org/W6684921986","https://openalex.org/W6686207219","https://openalex.org/W6720501231","https://openalex.org/W6738700159","https://openalex.org/W6745074091","https://openalex.org/W6745274615","https://openalex.org/W6745420753","https://openalex.org/W6770858630","https://openalex.org/W6772921146"],"related_works":["https://openalex.org/W2965591307","https://openalex.org/W1884601587","https://openalex.org/W3183112146","https://openalex.org/W2575705757","https://openalex.org/W3205641566","https://openalex.org/W3133865035","https://openalex.org/W2792868485","https://openalex.org/W2120982521","https://openalex.org/W3111672163","https://openalex.org/W2209580443","https://openalex.org/W2589275886","https://openalex.org/W140788646","https://openalex.org/W153292794","https://openalex.org/W3131470809","https://openalex.org/W2951072909","https://openalex.org/W2063471043","https://openalex.org/W3130717831","https://openalex.org/W904535523","https://openalex.org/W3133085002","https://openalex.org/W3157623683"],"abstract_inverted_index":{"Contact-rich":[0],"manipulation":[1],"tasks":[2],"in":[3,37,111],"unstructured":[4],"environments":[5],"often":[6],"require":[7],"both":[8],"haptic":[9],"and":[10,65,100,113],"visual":[11],"feedback.":[12],"However,":[13],"it":[14],"is":[15],"non-trivial":[16],"to":[17,49,55,61,77,105],"manually":[18],"design":[19],"a":[20,63,91,115],"robot":[21],"controller":[22],"that":[23],"combines":[24],"modalities":[25],"with":[26],"very":[27],"different":[28,97],"characteristics.":[29],"While":[30],"deep":[31],"reinforcement":[32],"learning":[33,38],"has":[34],"shown":[35],"success":[36],"control":[39],"policies":[40],"for":[41],"high-dimensional":[42],"inputs,":[43,71],"these":[44],"algorithms":[45],"are":[46],"generally":[47],"intractable":[48],"deploy":[50],"on":[51,90,114],"real":[52,116],"robots":[53],"due":[54],"sample":[56,80],"complexity.":[57],"We":[58,86,108],"use":[59],"self-supervision":[60],"learn":[62],"compact":[64],"multimodal":[66],"representation":[67],"of":[68,82],"our":[69,83,88],"sensory":[70],"which":[72],"can":[73],"then":[74],"be":[75],"used":[76],"improve":[78],"the":[79],"efficiency":[81],"policy":[84],"learning.":[85],"evaluate":[87],"method":[89],"peg":[92],"insertion":[93],"task,":[94],"generalizing":[95],"over":[96],"geometry,":[98],"configurations,":[99],"clearances,":[101],"while":[102],"being":[103],"robust":[104],"external":[106],"perturbations.":[107],"present":[109],"results":[110],"simulation":[112],"robot.":[117]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":9},{"year":2019,"cited_by_count":7}],"updated_date":"2026-03-11T14:59:36.786465","created_date":"2025-10-10T00:00:00"}