{"id":"https://openalex.org/W4312289535","doi":"https://doi.org/10.1109/iros47612.2022.9982241","title":"COMPASS: Contrastive Multimodal Pretraining for Autonomous Systems","display_name":"COMPASS: Contrastive Multimodal Pretraining for Autonomous Systems","publication_year":2022,"publication_date":"2022-10-23","ids":{"openalex":"https://openalex.org/W4312289535","doi":"https://doi.org/10.1109/iros47612.2022.9982241"},"language":"en","primary_location":{"id":"doi:10.1109/iros47612.2022.9982241","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iros47612.2022.9982241","pdf_url":null,"source":{"id":"https://openalex.org/S4363607704","display_name":"2022 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102021059","display_name":"Shuang Ma","orcid":"https://orcid.org/0000-0002-9870-5492"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["GB","US"],"is_corresponding":true,"raw_author_name":"Shuang Ma","raw_affiliation_strings":["Microsoft Redmond,WA","Microsoft Redmond, WA"],"affiliations":[{"raw_affiliation_string":"Microsoft Redmond,WA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050296324","display_name":"Sai Vemprala","orcid":"https://orcid.org/0000-0001-7554-5417"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Sai Vemprala","raw_affiliation_strings":["Microsoft Redmond,WA","Microsoft Redmond, WA"],"affiliations":[{"raw_affiliation_string":"Microsoft Redmond,WA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101634794","display_name":"Wenshan Wang","orcid":"https://orcid.org/0000-0002-4488-5619"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wenshan Wang","raw_affiliation_strings":["Carnegie Mellon University Pittsburgh,PA","Carnegie Mellon University Pittsburgh, PA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh,PA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002351623","display_name":"Jayesh K. Gupta","orcid":"https://orcid.org/0000-0002-4742-9942"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Jayesh K. Gupta","raw_affiliation_strings":["Microsoft Redmond,WA","Microsoft Redmond, WA"],"affiliations":[{"raw_affiliation_string":"Microsoft Redmond,WA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100566791","display_name":"Yale Song","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Yale Song","raw_affiliation_strings":["Microsoft Redmond,WA","Microsoft Redmond, WA"],"affiliations":[{"raw_affiliation_string":"Microsoft Redmond,WA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091840052","display_name":"Daniel McDufft","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Daniel McDufft","raw_affiliation_strings":["Microsoft Redmond,WA","Microsoft Redmond, WA"],"affiliations":[{"raw_affiliation_string":"Microsoft Redmond,WA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101933711","display_name":"Ashish Kapoor","orcid":"https://orcid.org/0009-0004-3764-8449"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Ashish Kapoor","raw_affiliation_strings":["Microsoft Redmond,WA","Microsoft Redmond, WA"],"affiliations":[{"raw_affiliation_string":"Microsoft Redmond,WA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5102021059"],"corresponding_institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"],"apc_list":null,"apc_paid":null,"fwci":0.5997,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.77071154,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1000","last_page":"1007"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10191","display_name":"Robotics and Sensor-Based Localization","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/2202","display_name":"Aerospace Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/compass","display_name":"Compass","score":0.8995298147201538},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7534064054489136},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6008660197257996},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.4280340075492859},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4105718731880188},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3410668671131134},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3220604658126831},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.15509918332099915}],"concepts":[{"id":"https://openalex.org/C2778361833","wikidata":"https://www.wikidata.org/wiki/Q34735","display_name":"Compass","level":2,"score":0.8995298147201538},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7534064054489136},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6008660197257996},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.4280340075492859},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4105718731880188},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3410668671131134},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3220604658126831},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.15509918332099915},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/iros47612.2022.9982241","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iros47612.2022.9982241","pdf_url":null,"source":{"id":"https://openalex.org/S4363607704","display_name":"2022 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":78,"referenced_works":["https://openalex.org/W1612997784","https://openalex.org/W1870947790","https://openalex.org/W2082971343","https://openalex.org/W2115579991","https://openalex.org/W2123086804","https://openalex.org/W2292391751","https://openalex.org/W2598706937","https://openalex.org/W2619383789","https://openalex.org/W2767290858","https://openalex.org/W2772409766","https://openalex.org/W2794258351","https://openalex.org/W2798302276","https://openalex.org/W2811406147","https://openalex.org/W2816936748","https://openalex.org/W2830339951","https://openalex.org/W2842511635","https://openalex.org/W2890124889","https://openalex.org/W2896457183","https://openalex.org/W2946165673","https://openalex.org/W2962795401","https://openalex.org/W2962934715","https://openalex.org/W2962960500","https://openalex.org/W2963149042","https://openalex.org/W2963583471","https://openalex.org/W2963689319","https://openalex.org/W2963782415","https://openalex.org/W2964314455","https://openalex.org/W2966287654","https://openalex.org/W2966715458","https://openalex.org/W2968095426","https://openalex.org/W2968354794","https://openalex.org/W2969101565","https://openalex.org/W2975357369","https://openalex.org/W2985076077","https://openalex.org/W2998356391","https://openalex.org/W3001279689","https://openalex.org/W3003957707","https://openalex.org/W3009205145","https://openalex.org/W3034419097","https://openalex.org/W3035056458","https://openalex.org/W3035060554","https://openalex.org/W3037309139","https://openalex.org/W3037338873","https://openalex.org/W3037784242","https://openalex.org/W3040041096","https://openalex.org/W3091588028","https://openalex.org/W3098994456","https://openalex.org/W3099342433","https://openalex.org/W3106440972","https://openalex.org/W3108655343","https://openalex.org/W3115293622","https://openalex.org/W3129860168","https://openalex.org/W3132270109","https://openalex.org/W3168822201","https://openalex.org/W3208092997","https://openalex.org/W4288281388","https://openalex.org/W4292779060","https://openalex.org/W4295927003","https://openalex.org/W4297808394","https://openalex.org/W4299811618","https://openalex.org/W6685670348","https://openalex.org/W6752823625","https://openalex.org/W6755207826","https://openalex.org/W6759097295","https://openalex.org/W6765843572","https://openalex.org/W6766904570","https://openalex.org/W6767853649","https://openalex.org/W6772383348","https://openalex.org/W6774314701","https://openalex.org/W6775186109","https://openalex.org/W6775224297","https://openalex.org/W6775634482","https://openalex.org/W6778883912","https://openalex.org/W6779326418","https://openalex.org/W6779809370","https://openalex.org/W6780151221","https://openalex.org/W6780294235","https://openalex.org/W6780443396"],"related_works":["https://openalex.org/W2206872286","https://openalex.org/W2382913035","https://openalex.org/W2113447385","https://openalex.org/W2391335452","https://openalex.org/W2360876323","https://openalex.org/W2122695655","https://openalex.org/W2109954054","https://openalex.org/W2642723128","https://openalex.org/W1970764900","https://openalex.org/W3196817267"],"abstract_inverted_index":{"Learning":[0],"representations":[1,131],"that":[2,132,169],"generalize":[3,179],"across":[4],"tasks":[5,47],"and":[6,72,89,102,114,141,154,162,176,183],"domains":[7],"is":[8],"challenging":[9],"yet":[10],"necessary":[11,134],"for":[12,61,86],"autonomous":[13,87],"systems.":[14],"Although":[15],"task-driven":[16],"approaches":[17],"are":[18,100],"appealing,":[19],"de-signing":[20],"models":[21,71,133],"specific":[22],"to":[23,65,180],"each":[24,125],"application":[25],"can":[26,171,177,195],"be":[27,196],"difficult":[28],"in":[29,48,124],"the":[30,53,67,83,90],"face":[31],"of":[32,69,92],"limited":[33],"data,":[34],"especially":[35],"when":[36],"dealing":[37],"with":[38],"highly":[39],"variable":[40],"multimodal":[41,79,98,122,149],"input":[42],"spaces":[43],"arising":[44],"from":[45,121],"different":[46,49,93],"environments.":[50],"We":[51,143],"introduce":[52],"first":[54],"general-purpose":[55],"pretraining":[56,74],"pipeline,":[57],"COntrastive":[58],"Multimodal":[59],"Pretraining":[60],"AutonomouS":[62],"Systems":[63],"(COMPASS),":[64],"overcome":[66],"limitations":[68],"task-specific":[70],"existing":[73],"approaches.":[75],"COMPASS":[76,128,145,170],"constructs":[77],"a":[78,110,115,147],"graph":[80],"by":[81],"considering":[82],"essential":[84],"information":[85,135],"systems":[88],"proper-ties":[91],"modalities.":[94],"Through":[95],"this":[96],"graph,":[97],"signals":[99],"connected":[101],"mapped":[103],"into":[104],"two":[105],"factorized":[106],"spatio-temporal":[107],"latent":[108,126],"spaces:":[109],"\u201cmotion":[111],"pattern":[112],"space\u201d":[113],"\u201ccurrent":[116],"state":[117,130],"space.\u201d":[118],"By":[119],"learning":[120],"correspondences":[123],"space,":[127],"creates":[129],"such":[136],"as":[137],"temporal":[138],"dynamics,":[139],"geometry,":[140],"semantics.":[142],"pretrain":[144],"on":[146,157],"large-scale":[148],"simulation":[150],"dataset":[151],"TartanAir":[152],"[1]":[153],"evaluate":[155],"it":[156],"drone":[158],"navigation,":[159],"vehicle":[160],"racing,":[161],"visual":[163],"odometry":[164],"tasks.":[165],"The":[166],"experiments":[167],"indicate":[168],"tackle":[172],"all":[173],"three":[174],"scenarios":[175],"also":[178],"unseen":[181],"environments":[182],"real-world":[184],"data.":[185],"<sup":[186,189],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[187,190],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">1</sup>":[188,191],"Our":[192],"code":[193],"implementation":[194],"found":[197],"at":[198],"https://github.com/microsoft/COMPASS":[199]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}