{"id":"https://openalex.org/W7105605013","doi":"https://doi.org/10.1109/tro.2025.3631816","title":"A Survey on Deep Generative Models for Robot Learning From Multimodal Demonstrations","display_name":"A Survey on Deep Generative Models for Robot Learning From Multimodal Demonstrations","publication_year":2025,"publication_date":"2025-11-13","ids":{"openalex":"https://openalex.org/W7105605013","doi":"https://doi.org/10.1109/tro.2025.3631816"},"language":null,"primary_location":{"id":"doi:10.1109/tro.2025.3631816","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tro.2025.3631816","pdf_url":null,"source":{"id":"https://openalex.org/S144620930","display_name":"IEEE Transactions on Robotics","issn_l":"1552-3098","issn":["1552-3098","1941-0468"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Robotics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Julen Urain","orcid":"https://orcid.org/0000-0003-1135-6654"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Julen Urain","raw_affiliation_strings":["META Fundamental AI Research (FAIR), Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"META Fundamental AI Research (FAIR), Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ajay Mandlekar","orcid":"https://orcid.org/0009-0008-5448-5011"},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ajay Mandlekar","raw_affiliation_strings":["NVIDIA AI, Santa Clara, CA, USA"],"affiliations":[{"raw_affiliation_string":"NVIDIA AI, Santa Clara, CA, USA","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yilun Du","orcid":null},"institutions":[{"id":"https://openalex.org/I2801851002","display_name":"Harvard University Press","ror":"https://ror.org/006v7bf86","country_code":"US","type":"other","lineage":["https://openalex.org/I136199984","https://openalex.org/I2801851002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yilun Du","raw_affiliation_strings":["Harvard University, Cambridge, MA, USA"],"affiliations":[{"raw_affiliation_string":"Harvard University, Cambridge, MA, USA","institution_ids":["https://openalex.org/I2801851002"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Nur Muhammad \u201cMahi\u201d Shafiullah","orcid":null},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nur Muhammad \u201cMahi\u201d Shafiullah","raw_affiliation_strings":["Berkeley AI Research (BAIR), Berkeley, CA, USA"],"affiliations":[{"raw_affiliation_string":"Berkeley AI Research (BAIR), Berkeley, CA, USA","institution_ids":["https://openalex.org/I134446601"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Danfei Xu","orcid":"https://orcid.org/0000-0002-8744-3861"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Danfei Xu","raw_affiliation_strings":["School of Interactive Computing, Georgia Tech, Atlanta, GA, USA"],"affiliations":[{"raw_affiliation_string":"School of Interactive Computing, Georgia Tech, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Katerina Fragkiadaki","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Katerina Fragkiadaki","raw_affiliation_strings":["Machine Learning Department in Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Machine Learning Department in Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Georgia Chalvatzaki","orcid":"https://orcid.org/0000-0002-5055-199X"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technische Universit\u00e4t Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Georgia Chalvatzaki","raw_affiliation_strings":["Computer Science Department of the Technical University of Darmstadt, Darmstadt, Germany"],"affiliations":[{"raw_affiliation_string":"Computer Science Department of the Technical University of Darmstadt, Darmstadt, Germany","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"last","author":{"id":null,"display_name":"Jan Peters","orcid":"https://orcid.org/0000-0002-5266-8091"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technische Universit\u00e4t Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Jan Peters","raw_affiliation_strings":["Computer Science Department, Technical University of Darmstadt, Darmstadt, Germany"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Technical University of Darmstadt, Darmstadt, Germany","institution_ids":["https://openalex.org/I31512782"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210114444"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.58157565,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"42","issue":null,"first_page":"60","last_page":"79"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.3361999988555908,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.3361999988555908,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.2646999955177307,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.20919999480247498,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.703499972820282},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.605400025844574},{"id":"https://openalex.org/keywords/grasp","display_name":"GRASP","score":0.560699999332428},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.553600013256073},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.5353999733924866},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.450300008058548},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.40070000290870667},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.3709999918937683}],"concepts":[{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7170000076293945},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.703499972820282},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6754000186920166},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.605400025844574},{"id":"https://openalex.org/C171268870","wikidata":"https://www.wikidata.org/wiki/Q1486676","display_name":"GRASP","level":2,"score":0.560699999332428},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.553600013256073},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.5353999733924866},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.450300008058548},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.429500013589859},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.40070000290870667},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.3709999918937683},{"id":"https://openalex.org/C126388530","wikidata":"https://www.wikidata.org/wiki/Q1131737","display_name":"Imitation","level":2,"score":0.3463999927043915},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.3425000011920929},{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.2897999882698059},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.2870999872684479},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.28439998626708984},{"id":"https://openalex.org/C184408114","wikidata":"https://www.wikidata.org/wiki/Q1502022","display_name":"Generative Design","level":3,"score":0.28439998626708984},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.27149999141693115},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.2644999921321869}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tro.2025.3631816","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tro.2025.3631816","pdf_url":null,"source":{"id":"https://openalex.org/S144620930","display_name":"IEEE Transactions on Robotics","issn_l":"1552-3098","issn":["1552-3098","1941-0468"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Robotics","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":110,"referenced_works":["https://openalex.org/W52153049","https://openalex.org/W1483373940","https://openalex.org/W1909846898","https://openalex.org/W1986014385","https://openalex.org/W1994648061","https://openalex.org/W2013035813","https://openalex.org/W2016418141","https://openalex.org/W2019965290","https://openalex.org/W2053910308","https://openalex.org/W2099893201","https://openalex.org/W2112474089","https://openalex.org/W2116064496","https://openalex.org/W2128677288","https://openalex.org/W2128990851","https://openalex.org/W2129202194","https://openalex.org/W2164032318","https://openalex.org/W2166302491","https://openalex.org/W2182977695","https://openalex.org/W2419216244","https://openalex.org/W2500624988","https://openalex.org/W2545129501","https://openalex.org/W2567455162","https://openalex.org/W2584986912","https://openalex.org/W2604382266","https://openalex.org/W2885546722","https://openalex.org/W2886316146","https://openalex.org/W2925241625","https://openalex.org/W2962793652","https://openalex.org/W2963033241","https://openalex.org/W2963092423","https://openalex.org/W2963150697","https://openalex.org/W2963341956","https://openalex.org/W2963351448","https://openalex.org/W2963439114","https://openalex.org/W2963516936","https://openalex.org/W2963634205","https://openalex.org/W2963689319","https://openalex.org/W2966615203","https://openalex.org/W2967842640","https://openalex.org/W2967895468","https://openalex.org/W2986303149","https://openalex.org/W2994446013","https://openalex.org/W3003310656","https://openalex.org/W3007769740","https://openalex.org/W3016361525","https://openalex.org/W3039055967","https://openalex.org/W3039737909","https://openalex.org/W3090369311","https://openalex.org/W3090584485","https://openalex.org/W3090814639","https://openalex.org/W3091470805","https://openalex.org/W3091677803","https://openalex.org/W3101780148","https://openalex.org/W3120441392","https://openalex.org/W3132689247","https://openalex.org/W3134231938","https://openalex.org/W3167771209","https://openalex.org/W3174849255","https://openalex.org/W3175388581","https://openalex.org/W3183673520","https://openalex.org/W3206683563","https://openalex.org/W3206916018","https://openalex.org/W3207181464","https://openalex.org/W3207837114","https://openalex.org/W3208129034","https://openalex.org/W3212320963","https://openalex.org/W4200108700","https://openalex.org/W4200150166","https://openalex.org/W4211008118","https://openalex.org/W4214813851","https://openalex.org/W4283787240","https://openalex.org/W4283788970","https://openalex.org/W4285102268","https://openalex.org/W4285102336","https://openalex.org/W4293498911","https://openalex.org/W4312282783","https://openalex.org/W4312363187","https://openalex.org/W4312610897","https://openalex.org/W4312824283","https://openalex.org/W4312933868","https://openalex.org/W4367721889","https://openalex.org/W4382049667","https://openalex.org/W4383097668","https://openalex.org/W4383108224","https://openalex.org/W4383108296","https://openalex.org/W4383108731","https://openalex.org/W4383109488","https://openalex.org/W4385403811","https://openalex.org/W4385430497","https://openalex.org/W4385430588","https://openalex.org/W4385430674","https://openalex.org/W4385430678","https://openalex.org/W4385430679","https://openalex.org/W4386075980","https://openalex.org/W4386076407","https://openalex.org/W4389665529","https://openalex.org/W4389665935","https://openalex.org/W4393241407","https://openalex.org/W4401414275","https://openalex.org/W4401414844","https://openalex.org/W4401415186","https://openalex.org/W4401417123","https://openalex.org/W4402354007","https://openalex.org/W4402354092","https://openalex.org/W4402354093","https://openalex.org/W4402354140","https://openalex.org/W4402354171","https://openalex.org/W4402727783","https://openalex.org/W4402890475","https://openalex.org/W4404307277"],"related_works":[],"abstract_inverted_index":{"Learning":[0],"from":[1,12,156],"Demonstrations,":[2],"the":[3,18,25,69,84,102,107,117,123,143,167,175,185,188,194,197,203],"field":[4],"that":[5,50,122],"proposes":[6],"to":[7,61,82,94,159,192],"learn":[8],"robot":[9,70],"behavior":[10],"models":[11,49,81,112,121,152,173,217],"data,":[13],"is":[14,174],"gaining":[15],"popularity":[16],"with":[17],"emergence":[19],"of":[20,64,86,101,109,120,146,166,171,178,196,210],"deep":[21,79,110,150,215],"generative":[22,80,111,137,151,172,216],"models.":[23,199],"Although":[24],"problem":[26],"has":[27,73,125,190],"been":[28,154],"studied":[29],"for":[30,213],"years":[31],"under":[32],"names":[33],"such":[34,127],"as":[35,128],"Imitation":[36],"Learning,":[37,43],"Behavioral":[38],"Cloning,":[39],"or":[40,57,136,162],"Inverse":[41],"Reinforcement":[42],"classical":[44],"methods":[45],"have":[46,153],"relied":[47],"on":[48],"don't":[51,58],"capture":[52,83],"complex":[53],"data":[54],"distributions":[55],"well":[56,60],"scale":[59],"large":[62,87],"numbers":[63],"demonstrations.":[65],"In":[66,89,180],"recent":[67],"years,":[68],"learning":[71,214],"community":[72,124,189],"shown":[74],"increasing":[75],"interest":[76],"in":[77,106,113,148,218],"using":[78],"complexity":[85],"datasets.":[88],"this":[90],"survey,":[91,182],"we":[92,183,201],"aim":[93],"provide":[95],"a":[96,208],"unified":[97],"and":[98,206],"comprehensive":[99],"review":[100,184],"last":[103],"year's":[104],"progress":[105],"use":[108],"robotics.":[114,219],"We":[115,140],"present":[116,142],"different":[118,144,186],"types":[119,145],"explored,":[126],"energy-based":[129],"models,":[130,132],"diffusion":[131],"action":[133],"value":[134],"maps,":[135],"adversarial":[138],"networks.":[139],"also":[141],"applications":[147],"which":[149],"used,":[155],"grasp":[157],"generation":[158,161],"trajectory":[160],"cost":[163],"learning.":[164],"One":[165],"most":[168],"important":[169],"elements":[170],"generalization":[176,195],"out":[177],"distributions.":[179],"our":[181],"decisions":[187],"made":[191],"improve":[193],"learned":[198],"Finally,":[200],"highlight":[202],"research":[204],"challenges":[205],"propose":[207],"number":[209],"future":[211],"directions":[212]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-11-13T00:00:00"}