{"id":"https://openalex.org/W3033090297","doi":"https://doi.org/10.1177/0278364920917755","title":"Multimodal estimation and communication of latent semantic knowledge for robust execution of robot instructions","display_name":"Multimodal estimation and communication of latent semantic knowledge for robust execution of robot instructions","publication_year":2020,"publication_date":"2020-06-05","ids":{"openalex":"https://openalex.org/W3033090297","doi":"https://doi.org/10.1177/0278364920917755","mag":"3033090297"},"language":"en","primary_location":{"id":"doi:10.1177/0278364920917755","is_oa":false,"landing_page_url":"https://doi.org/10.1177/0278364920917755","pdf_url":null,"source":{"id":"https://openalex.org/S73484101","display_name":"The International Journal of Robotics Research","issn_l":"0278-3649","issn":["0278-3649","1741-3176"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The International Journal of Robotics Research","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://hdl.handle.net/1721.1/130573","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016548903","display_name":"Jacob Arkin","orcid":"https://orcid.org/0000-0002-1074-9248"},"institutions":[{"id":"https://openalex.org/I5388228","display_name":"University of Rochester","ror":"https://ror.org/022kthw22","country_code":"US","type":"education","lineage":["https://openalex.org/I5388228"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jacob Arkin","raw_affiliation_strings":["Robotics and Artificial Intelligence Laboratory, University of Rochester, USA"],"affiliations":[{"raw_affiliation_string":"Robotics and Artificial Intelligence Laboratory, University of Rochester, USA","institution_ids":["https://openalex.org/I5388228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074295573","display_name":"Daehyung Park","orcid":"https://orcid.org/0000-0002-1287-9433"},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Daehyung Park","raw_affiliation_strings":["Computer Science & Artificial Intelligence Laboratory, Massachusetts Institute of Technology, USA"],"affiliations":[{"raw_affiliation_string":"Computer Science & Artificial Intelligence Laboratory, Massachusetts Institute of Technology, USA","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041779935","display_name":"Subhro Roy","orcid":null},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Subhro Roy","raw_affiliation_strings":["Computer Science & Artificial Intelligence Laboratory, Massachusetts Institute of Technology, USA"],"affiliations":[{"raw_affiliation_string":"Computer Science & Artificial Intelligence Laboratory, Massachusetts Institute of Technology, USA","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103153703","display_name":"Matthew R. Walter","orcid":"https://orcid.org/0000-0003-1425-6050"},"institutions":[{"id":"https://openalex.org/I160992636","display_name":"Toyota Technological Institute at Chicago","ror":"https://ror.org/02sn5gb64","country_code":"US","type":"education","lineage":["https://openalex.org/I160992636"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew R Walter","raw_affiliation_strings":["Robot Intelligence through Perception Laboratory, Toyota Technological Institute at Chicago, USA"],"affiliations":[{"raw_affiliation_string":"Robot Intelligence through Perception Laboratory, Toyota Technological Institute at Chicago, USA","institution_ids":["https://openalex.org/I160992636"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004980809","display_name":"Nicholas Roy","orcid":"https://orcid.org/0000-0002-4277-2928"},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nicholas Roy","raw_affiliation_strings":["Computer Science & Artificial Intelligence Laboratory, Massachusetts Institute of Technology, USA"],"affiliations":[{"raw_affiliation_string":"Computer Science & Artificial Intelligence Laboratory, Massachusetts Institute of Technology, USA","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101411184","display_name":"Thomas M. Howard","orcid":"https://orcid.org/0000-0002-8270-0806"},"institutions":[{"id":"https://openalex.org/I5388228","display_name":"University of Rochester","ror":"https://ror.org/022kthw22","country_code":"US","type":"education","lineage":["https://openalex.org/I5388228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Thomas M Howard","raw_affiliation_strings":["Robotics and Artificial Intelligence Laboratory, University of Rochester, USA"],"affiliations":[{"raw_affiliation_string":"Robotics and Artificial Intelligence Laboratory, University of Rochester, USA","institution_ids":["https://openalex.org/I5388228"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5002249903","display_name":"Rohan Paul","orcid":"https://orcid.org/0000-0002-6733-1766"},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]},{"id":"https://openalex.org/I68891433","display_name":"Indian Institute of Technology Delhi","ror":"https://ror.org/049tgcd06","country_code":"IN","type":"education","lineage":["https://openalex.org/I68891433"]}],"countries":["IN","US"],"is_corresponding":true,"raw_author_name":"Rohan Paul","raw_affiliation_strings":["Computer Science & Artificial Intelligence Laboratory, Massachusetts Institute of Technology, USA","Department of Computer Science and Engineering, Indian Institute of Technology Delhi, India"],"affiliations":[{"raw_affiliation_string":"Computer Science & Artificial Intelligence Laboratory, Massachusetts Institute of Technology, USA","institution_ids":["https://openalex.org/I63966007"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, Indian Institute of Technology Delhi, India","institution_ids":["https://openalex.org/I68891433"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5002249903","https://openalex.org/A5016548903","https://openalex.org/A5074295573"],"corresponding_institution_ids":["https://openalex.org/I5388228","https://openalex.org/I63966007","https://openalex.org/I68891433"],"apc_list":null,"apc_paid":null,"fwci":2.649,"has_fulltext":false,"cited_by_count":40,"citation_normalized_percentile":{"value":0.91680345,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"39","issue":"10-11","first_page":"1279","last_page":"1304"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7836309671401978},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.6296912431716919},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5933359265327454},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.493778795003891},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4341944456100464},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4271070957183838},{"id":"https://openalex.org/keywords/semantic-mapping","display_name":"Semantic mapping","score":0.4174199104309082},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.4118577539920807},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3766871690750122}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7836309671401978},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.6296912431716919},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5933359265327454},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.493778795003891},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4341944456100464},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4271070957183838},{"id":"https://openalex.org/C2775955345","wikidata":"https://www.wikidata.org/wiki/Q7449071","display_name":"Semantic mapping","level":2,"score":0.4174199104309082},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.4118577539920807},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3766871690750122},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1177/0278364920917755","is_oa":false,"landing_page_url":"https://doi.org/10.1177/0278364920917755","pdf_url":null,"source":{"id":"https://openalex.org/S73484101","display_name":"The International Journal of Robotics Research","issn_l":"0278-3649","issn":["0278-3649","1741-3176"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The International Journal of Robotics Research","raw_type":"journal-article"},{"id":"pmh:oai:dspace.mit.edu:1721.1/130573","is_oa":true,"landing_page_url":"https://hdl.handle.net/1721.1/130573","pdf_url":null,"source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Prof. Roy","raw_type":"http://purl.org/eprint/type/JournalArticle"}],"best_oa_location":{"id":"pmh:oai:dspace.mit.edu:1721.1/130573","is_oa":true,"landing_page_url":"https://hdl.handle.net/1721.1/130573","pdf_url":null,"source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Prof. Roy","raw_type":"http://purl.org/eprint/type/JournalArticle"},"sustainable_development_goals":[{"score":0.5699999928474426,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G8000823835","display_name":null,"funder_award_id":"LP- C000765-SR","funder_id":"https://openalex.org/F4320315934","funder_display_name":"Toyota Research Institute"}],"funders":[{"id":"https://openalex.org/F4320306994","display_name":"Lockheed Martin","ror":"https://ror.org/026er9r08"},{"id":"https://openalex.org/F4320315934","display_name":"Toyota Research Institute","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":70,"referenced_works":["https://openalex.org/W48884151","https://openalex.org/W1173687456","https://openalex.org/W1493086929","https://openalex.org/W1706049732","https://openalex.org/W1880262756","https://openalex.org/W1933065844","https://openalex.org/W1949907236","https://openalex.org/W1980043766","https://openalex.org/W1991374750","https://openalex.org/W1993302780","https://openalex.org/W1999988626","https://openalex.org/W2003170434","https://openalex.org/W2007820193","https://openalex.org/W2010158189","https://openalex.org/W2013511564","https://openalex.org/W2031500961","https://openalex.org/W2054228171","https://openalex.org/W2060088703","https://openalex.org/W2068537423","https://openalex.org/W2075654868","https://openalex.org/W2107019937","https://openalex.org/W2109388348","https://openalex.org/W2122410182","https://openalex.org/W2125838338","https://openalex.org/W2127498532","https://openalex.org/W2142315290","https://openalex.org/W2149606722","https://openalex.org/W2150692719","https://openalex.org/W2154764394","https://openalex.org/W2161002933","https://openalex.org/W2196779496","https://openalex.org/W2199571216","https://openalex.org/W2229480318","https://openalex.org/W2236233024","https://openalex.org/W2250539671","https://openalex.org/W2251079016","https://openalex.org/W2251512949","https://openalex.org/W2295357975","https://openalex.org/W2296455319","https://openalex.org/W2336416123","https://openalex.org/W2396269731","https://openalex.org/W2410925190","https://openalex.org/W2462848072","https://openalex.org/W2467155776","https://openalex.org/W2489434015","https://openalex.org/W2530850764","https://openalex.org/W2583360688","https://openalex.org/W2615497679","https://openalex.org/W2764255392","https://openalex.org/W2797862627","https://openalex.org/W2810720462","https://openalex.org/W2889756642","https://openalex.org/W2902699874","https://openalex.org/W2930957955","https://openalex.org/W2962781380","https://openalex.org/W2962858109","https://openalex.org/W2962905474","https://openalex.org/W2963109634","https://openalex.org/W2963244312","https://openalex.org/W2963800628","https://openalex.org/W2976490211","https://openalex.org/W2990138404","https://openalex.org/W3110909889","https://openalex.org/W4210700398","https://openalex.org/W4212863985","https://openalex.org/W4238869533","https://openalex.org/W4255895325","https://openalex.org/W4310895557","https://openalex.org/W4313017144","https://openalex.org/W6600480908"],"related_works":["https://openalex.org/W17155033","https://openalex.org/W3207760230","https://openalex.org/W1496222301","https://openalex.org/W4312814274","https://openalex.org/W1590307681","https://openalex.org/W2536018345","https://openalex.org/W4285370786","https://openalex.org/W2296488620","https://openalex.org/W3213722473","https://openalex.org/W2110944602"],"abstract_inverted_index":{"The":[0],"goal":[1],"of":[2,73,90,118,156,239],"this":[3],"article":[4],"is":[5,29],"to":[6,9,24,32,47,114,129,145,150,212,219,225,232],"enable":[7],"robots":[8,38],"perform":[10],"robust":[11,128],"task":[12],"execution":[13],"following":[14],"human":[15,152],"instructions":[16,119],"in":[17,50,125,159],"partially":[18,203],"observable":[19],"environments.":[20],"A":[21],"robot\u2019s":[22,237],"ability":[23,211],"interpret":[25],"and":[26,53,58,81,103,120,175,190,207,216,222],"execute":[27,121],"commands":[28],"fundamentally":[30],"tied":[31],"its":[33],"semantic":[34,62,88,205],"world":[35,63,112,162,183],"knowledge.":[36],"Commonly,":[37],"use":[39,72],"exteroceptive":[40],"sensors,":[41],"such":[42],"as":[43,84,153],"cameras":[44],"or":[45,132],"LiDAR,":[46],"detect":[48],"entities":[49],"the":[51,71,116,122,143,151,160,177,181,236],"workspace":[52],"infer":[54,115],"their":[55],"visual":[56,102],"properties":[57,64,89],"spatial":[59],"relationships.":[60],"However,":[61],"are":[65],"often":[66],"visually":[67],"imperceptible.":[68],"We":[69,92,192],"posit":[70],"non-exteroceptive":[74],"modalities":[75],"including":[76],"physical":[77],"proprioception,":[78],"factual":[79],"descriptions,":[80],"domain":[82],"knowledge":[83,100,147,217],"mechanisms":[85],"for":[86,180,197],"inferring":[87],"objects.":[91],"introduce":[93],"a":[94,107,126,139,154],"probabilistic":[95],"model":[96],"that":[97,141,170,199,229],"fuses":[98],"linguistic":[99,173],"with":[101,235],"haptic":[104],"observations":[105],"into":[106],"cumulative":[108],"belief":[109],"over":[110,202],"latent":[111],"attributes":[113],"meaning":[117],"instructed":[123],"tasks":[124,198],"manner":[127],"erroneous,":[130],"noisy,":[131],"contradictory":[133],"evidence.":[134],"In":[135],"addition,":[136],"we":[137,165],"provide":[138],"method":[140],"allows":[142],"robot":[144],"communicate":[146],"dissonance":[148],"back":[149],"means":[155],"correcting":[157],"errors":[158],"operator\u2019s":[161],"model.":[163],"Finally,":[164],"propose":[166],"an":[167],"efficient":[168],"framework":[169],"anticipates":[171],"possible":[172],"interactions":[174],"infers":[176],"associated":[178],"groundings":[179],"current":[182],"state,":[184],"thereby":[185],"bootstrapping":[186],"both":[187],"language":[188],"understanding":[189],"generation.":[191],"present":[193],"experiments":[194],"on":[195],"manipulators":[196],"require":[200],"inference":[201],"observed":[204,231],"properties,":[206],"evaluate":[208],"our":[209],"framework\u2019s":[210],"exploit":[213],"expressed":[214],"information":[215],"bases":[218],"facilitate":[220],"convergence,":[221],"generate":[223],"statements":[224],"correct":[226],"declared":[227],"facts":[228],"were":[230],"be":[233],"inconsistent":[234],"estimate":[238],"object":[240],"properties.":[241]},"counts_by_year":[{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":2}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}