{"id":"https://openalex.org/W4408355118","doi":"https://doi.org/10.1109/icassp49660.2025.10887717","title":"Interactive Robot Action Replanning using Multimodal LLM Trained from Human Demonstration Videos","display_name":"Interactive Robot Action Replanning using Multimodal LLM Trained from Human Demonstration Videos","publication_year":2025,"publication_date":"2025-03-12","ids":{"openalex":"https://openalex.org/W4408355118","doi":"https://doi.org/10.1109/icassp49660.2025.10887717"},"language":"en","primary_location":{"id":"doi:10.1109/icassp49660.2025.10887717","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp49660.2025.10887717","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2025 - 2025 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001601327","display_name":"Chiori Hori","orcid":"https://orcid.org/0000-0002-4201-7578"},"institutions":[{"id":"https://openalex.org/I4210159266","display_name":"Mitsubishi Electric (United States)","ror":"https://ror.org/053jnhe44","country_code":"US","type":"company","lineage":["https://openalex.org/I1306287861","https://openalex.org/I4210133125","https://openalex.org/I4210159266"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Chiori Hori","raw_affiliation_strings":["Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA","institution_ids":["https://openalex.org/I4210159266"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030254193","display_name":"Motonari Kambara","orcid":"https://orcid.org/0000-0002-1991-9119"},"institutions":[{"id":"https://openalex.org/I4210159266","display_name":"Mitsubishi Electric (United States)","ror":"https://ror.org/053jnhe44","country_code":"US","type":"company","lineage":["https://openalex.org/I1306287861","https://openalex.org/I4210133125","https://openalex.org/I4210159266"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Motonari Kambara","raw_affiliation_strings":["Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA","institution_ids":["https://openalex.org/I4210159266"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033744547","display_name":"Komei Sugiura","orcid":"https://orcid.org/0000-0002-0261-0510"},"institutions":[{"id":"https://openalex.org/I203951103","display_name":"Keio University","ror":"https://ror.org/02kn6nx58","country_code":"JP","type":"education","lineage":["https://openalex.org/I203951103"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Komei Sugiura","raw_affiliation_strings":["Keio University,Tokyo,Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Keio University,Tokyo,Japan","institution_ids":["https://openalex.org/I203951103"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064968116","display_name":"Kei Ota","orcid":"https://orcid.org/0000-0002-5744-3027"},"institutions":[{"id":"https://openalex.org/I4210159266","display_name":"Mitsubishi Electric (United States)","ror":"https://ror.org/053jnhe44","country_code":"US","type":"company","lineage":["https://openalex.org/I1306287861","https://openalex.org/I4210133125","https://openalex.org/I4210159266"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kei Ota","raw_affiliation_strings":["Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA","institution_ids":["https://openalex.org/I4210159266"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035187454","display_name":"Sameer Khurana","orcid":"https://orcid.org/0000-0002-3182-1085"},"institutions":[{"id":"https://openalex.org/I4210159266","display_name":"Mitsubishi Electric (United States)","ror":"https://ror.org/053jnhe44","country_code":"US","type":"company","lineage":["https://openalex.org/I1306287861","https://openalex.org/I4210133125","https://openalex.org/I4210159266"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sameer Khurana","raw_affiliation_strings":["Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA","institution_ids":["https://openalex.org/I4210159266"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062291876","display_name":"Siddarth Jain","orcid":null},"institutions":[{"id":"https://openalex.org/I4210159266","display_name":"Mitsubishi Electric (United States)","ror":"https://ror.org/053jnhe44","country_code":"US","type":"company","lineage":["https://openalex.org/I1306287861","https://openalex.org/I4210133125","https://openalex.org/I4210159266"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Siddarth Jain","raw_affiliation_strings":["Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA","institution_ids":["https://openalex.org/I4210159266"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055442775","display_name":"Radu Corcodel","orcid":null},"institutions":[{"id":"https://openalex.org/I4210159266","display_name":"Mitsubishi Electric (United States)","ror":"https://ror.org/053jnhe44","country_code":"US","type":"company","lineage":["https://openalex.org/I1306287861","https://openalex.org/I4210133125","https://openalex.org/I4210159266"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Radu Corcodel","raw_affiliation_strings":["Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA","institution_ids":["https://openalex.org/I4210159266"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010530960","display_name":"Devesh K. Jha","orcid":"https://orcid.org/0000-0002-7843-9545"},"institutions":[{"id":"https://openalex.org/I4210159266","display_name":"Mitsubishi Electric (United States)","ror":"https://ror.org/053jnhe44","country_code":"US","type":"company","lineage":["https://openalex.org/I1306287861","https://openalex.org/I4210133125","https://openalex.org/I4210159266"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Devesh Jha","raw_affiliation_strings":["Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA","institution_ids":["https://openalex.org/I4210159266"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062272788","display_name":"Diego Romeres","orcid":"https://orcid.org/0000-0002-8603-2438"},"institutions":[{"id":"https://openalex.org/I4210159266","display_name":"Mitsubishi Electric (United States)","ror":"https://ror.org/053jnhe44","country_code":"US","type":"company","lineage":["https://openalex.org/I1306287861","https://openalex.org/I4210133125","https://openalex.org/I4210159266"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Diego Romeres","raw_affiliation_strings":["Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA","institution_ids":["https://openalex.org/I4210159266"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076453358","display_name":"Jonathan Le Roux","orcid":"https://orcid.org/0000-0002-3451-171X"},"institutions":[{"id":"https://openalex.org/I4210159266","display_name":"Mitsubishi Electric (United States)","ror":"https://ror.org/053jnhe44","country_code":"US","type":"company","lineage":["https://openalex.org/I1306287861","https://openalex.org/I4210133125","https://openalex.org/I4210159266"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jonathan Le Roux","raw_affiliation_strings":["Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mitsubishi Electric Research Laboratories (MERL),Cambridge,MA,USA","institution_ids":["https://openalex.org/I4210159266"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5001601327"],"corresponding_institution_ids":["https://openalex.org/I4210159266"],"apc_list":null,"apc_paid":null,"fwci":3.2372,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.90744238,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.8633000254631042,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.8633000254631042,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.822700023651123,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12290","display_name":"Human Motion and Animation","score":0.8141000270843506,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7597215175628662},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.679940402507782},{"id":"https://openalex.org/keywords/human\u2013robot-interaction","display_name":"Human\u2013robot interaction","score":0.617935299873352},{"id":"https://openalex.org/keywords/action-recognition","display_name":"Action recognition","score":0.5973553657531738},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.5707271695137024},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5310718417167664},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5093191862106323},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.4353833496570587}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7597215175628662},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.679940402507782},{"id":"https://openalex.org/C145460709","wikidata":"https://www.wikidata.org/wiki/Q859951","display_name":"Human\u2013robot interaction","level":3,"score":0.617935299873352},{"id":"https://openalex.org/C2987834672","wikidata":"https://www.wikidata.org/wiki/Q4677630","display_name":"Action recognition","level":3,"score":0.5973553657531738},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5707271695137024},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5310718417167664},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5093191862106323},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4353833496570587},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp49660.2025.10887717","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp49660.2025.10887717","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2025 - 2025 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":21,"referenced_works":["https://openalex.org/W2007920248","https://openalex.org/W2250539671","https://openalex.org/W2909713961","https://openalex.org/W2952132648","https://openalex.org/W2963860638","https://openalex.org/W3096609285","https://openalex.org/W3196974791","https://openalex.org/W3207959086","https://openalex.org/W4312372834","https://openalex.org/W4383108457","https://openalex.org/W4385822471","https://openalex.org/W4386251907","https://openalex.org/W4391678022","https://openalex.org/W4399374221","https://openalex.org/W4401414856","https://openalex.org/W6755207826","https://openalex.org/W6791353385","https://openalex.org/W6801810553","https://openalex.org/W6810640255","https://openalex.org/W6811340617","https://openalex.org/W6849177959"],"related_works":["https://openalex.org/W1576128429","https://openalex.org/W2978665606","https://openalex.org/W4287179229","https://openalex.org/W3205513966","https://openalex.org/W3120459843","https://openalex.org/W4366547574","https://openalex.org/W2269464716","https://openalex.org/W3200191727","https://openalex.org/W189465620","https://openalex.org/W4366818884"],"abstract_inverted_index":{"Understanding":[0],"human":[1,39,65,108,121,172],"actions":[2,100],"could":[3,67],"allow":[4],"robots":[5],"to":[6,31,51,120],"perform":[7],"a":[8,102,139,146,193],"large":[9],"spectrum":[10],"of":[11,38,148,167,208],"complex":[12],"manipulation":[13],"tasks":[14],"and":[15,58,106,155,199],"make":[16],"collaboration":[17],"with":[18],"humans":[19],"easier.":[20],"Recently,":[21],"multimodal":[22,141],"scene":[23],"understanding":[24],"using":[25,96,138,196],"audio-visual":[26],"Transformers":[27],"has":[28],"been":[29],"used":[30],"generate":[32],"robot":[33,75,116,151,181],"action":[34,43,88,94,104,117,122,135,153,157,197],"sequences":[35,95,118,154],"from":[36,101],"videos":[37],"demonstrations.":[40],"However,":[41],"automatic":[42],"sequence":[44,198],"generation":[45],"is":[46],"not":[47],"always":[48],"perfect":[49],"due":[50],"the":[52,56,74,125,132,165,178,202],"distribution":[53],"gap":[54],"between":[55],"training":[57],"test":[59],"environments.":[60],"To":[61,163],"bridge":[62],"this":[63],"gap,":[64],"intervention":[66],"be":[68,79],"very":[69],"effective,":[70],"such":[71],"as":[72],"telling":[73],"agent":[76],"what":[77],"should":[78],"done.":[80],"Motivated":[81],"by":[82],"this,":[83],"we":[84,170],"propose":[85],"an":[86],"error-correction-based":[87],"replanning":[89,136,189],"approach":[90],"that":[91,185],"regenerates":[92],"better":[93],"(1)":[97],"automatically":[98,179],"generated":[99,180],"pretrained":[103],"generator":[105],"(2)":[107],"error-correction":[109],"in":[110,159,177,192,205],"natural":[111,160],"language.":[112],"We":[113,130],"collected":[114,171],"singlearm":[115],"aligned":[119],"instruction":[123],"for":[124],"cooking":[126],"video":[127],"dataset":[128],"YouCook2.":[129],"trained":[131,191],"proposed":[133,187],"errorcorrection-based":[134],"model":[137,143,190,204],"pre-trained":[140],"LLM":[142],"(AVBLIP-2),":[144],"generating":[145],"pair":[147],"(a)":[149],"single-arm":[150],"micro-step":[152],"(b)":[156],"descriptions":[158],"language":[161],"simultaneously.":[162],"assess":[164],"performance":[166],"error":[168],"correction,":[169],"feedback":[173],"on":[174],"correcting":[175],"errors":[176],"actions.":[182],"Experiments":[183],"show":[184],"our":[186],"interactive":[188],"multitask":[194],"manner":[195],"description":[200],"outperformed":[201],"baseline":[203],"all":[206],"types":[207],"scores.":[209]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-05-03T08:25:01.440150","created_date":"2025-10-10T00:00:00"}