{"id":"https://openalex.org/W4401415075","doi":"https://doi.org/10.1109/icra57147.2024.10610948","title":"Prompt, Plan, Perform: LLM-based Humanoid Control via Quantized Imitation Learning","display_name":"Prompt, Plan, Perform: LLM-based Humanoid Control via Quantized Imitation Learning","publication_year":2024,"publication_date":"2024-05-13","ids":{"openalex":"https://openalex.org/W4401415075","doi":"https://doi.org/10.1109/icra57147.2024.10610948"},"language":"en","primary_location":{"id":"doi:10.1109/icra57147.2024.10610948","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra57147.2024.10610948","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049574394","display_name":"Jingkai Sun","orcid":"https://orcid.org/0000-0002-1032-2957"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Jingkai Sun","raw_affiliation_strings":["The Hong Kong University of Science and Technology,Guangzhou,China"],"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology,Guangzhou,China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100381999","display_name":"Qiang Zhang","orcid":"https://orcid.org/0000-0003-3776-9799"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiang Zhang","raw_affiliation_strings":["The Hong Kong University of Science and Technology,Guangzhou,China"],"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology,Guangzhou,China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059407219","display_name":"Yiqun Duan","orcid":"https://orcid.org/0000-0003-1517-994X"},"institutions":[{"id":"https://openalex.org/I114017466","display_name":"University of Technology Sydney","ror":"https://ror.org/03f0f6041","country_code":"AU","type":"education","lineage":["https://openalex.org/I114017466"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Yiqun Duan","raw_affiliation_strings":["Australia Artificial Intelligence Institute University of Technology Sydney,Human | Centric AI Centre,Ultimo,Australia,2007"],"affiliations":[{"raw_affiliation_string":"Australia Artificial Intelligence Institute University of Technology Sydney,Human | Centric AI Centre,Ultimo,Australia,2007","institution_ids":["https://openalex.org/I114017466"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109664375","display_name":"Xiaoyang Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiaoyang Jiang","raw_affiliation_strings":["The Hong Kong University of Science and Technology,Guangzhou,China"],"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology,Guangzhou,China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100348801","display_name":"Chong Chen","orcid":"https://orcid.org/0000-0003-2800-4647"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chong Cheng","raw_affiliation_strings":["The Hong Kong University of Science and Technology,Guangzhou,China"],"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology,Guangzhou,China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100616654","display_name":"Renjing Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Renjing Xu","raw_affiliation_strings":["The Hong Kong University of Science and Technology,Guangzhou,China"],"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology,Guangzhou,China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5049574394"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.1152,"has_fulltext":false,"cited_by_count":27,"citation_normalized_percentile":{"value":0.97930992,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"16236","last_page":"16242"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10879","display_name":"Robotic Locomotion and Control","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7106227874755859},{"id":"https://openalex.org/keywords/plan","display_name":"Plan (archaeology)","score":0.6492021083831787},{"id":"https://openalex.org/keywords/imitation","display_name":"Imitation","score":0.6017606258392334},{"id":"https://openalex.org/keywords/humanoid-robot","display_name":"Humanoid robot","score":0.5604772567749023},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.5251032114028931},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.40253591537475586},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3795335590839386},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.13612034916877747},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.12028300762176514}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7106227874755859},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.6492021083831787},{"id":"https://openalex.org/C126388530","wikidata":"https://www.wikidata.org/wiki/Q1131737","display_name":"Imitation","level":2,"score":0.6017606258392334},{"id":"https://openalex.org/C60692881","wikidata":"https://www.wikidata.org/wiki/Q584529","display_name":"Humanoid robot","level":3,"score":0.5604772567749023},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.5251032114028931},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40253591537475586},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3795335590839386},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.13612034916877747},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.12028300762176514},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/icra57147.2024.10610948","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra57147.2024.10610948","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-143329","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-143329","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Zero hunger","score":0.4399999976158142,"id":"https://metadata.un.org/sdg/2"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W2056716515","https://openalex.org/W2124267516","https://openalex.org/W2580909119","https://openalex.org/W2736601468","https://openalex.org/W2884247313","https://openalex.org/W2962787969","https://openalex.org/W2964121592","https://openalex.org/W3011768656","https://openalex.org/W3128707967","https://openalex.org/W3147968035","https://openalex.org/W3168892396","https://openalex.org/W3171176697","https://openalex.org/W3181074849","https://openalex.org/W3207359859","https://openalex.org/W3213974477","https://openalex.org/W4207021923","https://openalex.org/W4210444794","https://openalex.org/W4212774754","https://openalex.org/W4224912544","https://openalex.org/W4229044820","https://openalex.org/W4285218426","https://openalex.org/W4285428875","https://openalex.org/W4293498911","https://openalex.org/W4310297571","https://openalex.org/W4311448205","https://openalex.org/W4312908419","https://openalex.org/W4368755539","https://openalex.org/W4383066216","https://openalex.org/W4383097638","https://openalex.org/W4383109478","https://openalex.org/W4383173602","https://openalex.org/W4384918448","https://openalex.org/W4384918756","https://openalex.org/W4385473486","https://openalex.org/W6718092244","https://openalex.org/W6741002519","https://openalex.org/W6752483423","https://openalex.org/W6753770476","https://openalex.org/W6754615820","https://openalex.org/W6791353385","https://openalex.org/W6793841834","https://openalex.org/W6800004206","https://openalex.org/W6801810553","https://openalex.org/W6809509765","https://openalex.org/W6810640255","https://openalex.org/W6810738896","https://openalex.org/W6839928859","https://openalex.org/W6845636941","https://openalex.org/W6854738657","https://openalex.org/W6854866820","https://openalex.org/W6855078748"],"related_works":["https://openalex.org/W4387497383","https://openalex.org/W3183948672","https://openalex.org/W3173606202","https://openalex.org/W3110381201","https://openalex.org/W2948807893","https://openalex.org/W2935909890","https://openalex.org/W2778153218","https://openalex.org/W2758277628","https://openalex.org/W1531601525","https://openalex.org/W1569471492"],"abstract_inverted_index":{"In":[0,87],"recent":[1],"years,":[2],"reinforcement":[3],"learning":[4,7,58,198],"and":[5,24,36,42,78,201,214],"imitation":[6,57],"have":[8],"shown":[9],"great":[10],"potential":[11],"for":[12,26,39,97],"controlling":[13],"humanoid":[14,160,193],"robots\u2019":[15],"motion.":[16],"However,":[17],"these":[18,47],"methods":[19],"typically":[20],"create":[21],"simulation":[22],"environments":[23],"rewards":[25],"specific":[27],"tasks,":[28],"resulting":[29],"in":[30,120,139,217],"the":[31,68,83,91,106,113,117,133,155,163,166,181,188],"requirements":[32],"of":[33,85,108,159,183],"multiple":[34],"policies":[35],"limited":[37],"capabilities":[38],"tackling":[40],"complex":[41],"unknown":[43],"tasks.":[44,220],"To":[45,123,180],"overcome":[46],"issues,":[48],"we":[49,89,127,148],"present":[50],"a":[51,75,94,121,196,204],"novel":[52,103],"approach":[53],"that":[54,153,191,209],"combines":[55],"adversarial":[56],"with":[59,74],"large":[60],"language":[61],"models":[62],"(LLMs).":[63],"This":[64,111],"innovative":[65],"method":[66,211],"enables":[67],"agent":[69,134,164],"to":[70,102,115,135,141],"learn":[71],"reusable":[72],"skills":[73,101],"single":[76,197],"policy":[77,199],"solve":[79],"zero-shot":[80],"tasks":[81,104],"under":[82],"guidance":[84],"LLMs.":[86,146],"particular,":[88],"utilize":[90],"LLM":[92,202],"as":[93,203],"strategic":[95],"planner":[96],"applying":[98],"previously":[99],"learned":[100],"through":[105],"comprehension":[107],"task-specific":[109],"prompts.":[110],"empowers":[112],"robot":[114],"perform":[116],"specified":[118],"actions":[119,138],"sequence.":[122],"improve":[124],"our":[125,184,210],"model,":[126],"incorporate":[128],"codebook-based":[129],"vector":[130],"quantization,":[131],"allowing":[132],"generate":[136],"suitable":[137],"response":[140],"unseen":[142],"textual":[143],"commands":[144],"from":[145],"Furthermore,":[147],"design":[149],"general":[150],"reward":[151],"functions":[152],"consider":[154],"distinct":[156],"motion":[157,167,219],"features":[158],"robots,":[161],"ensuring":[162],"imitates":[165],"data":[168],"while":[169],"maintaining":[170],"goal":[171],"orientation":[172],"without":[173],"additional":[174],"guiding":[175],"direction":[176],"approaches":[177],"or":[178],"policies.":[179],"best":[182],"knowledge,":[185],"this":[186],"is":[187],"first":[189],"framework":[190],"controls":[192],"robots":[194],"using":[195],"network":[200],"planner.":[205],"Extensive":[206],"experiments":[207],"demonstrate":[208],"exhibits":[212],"efficient":[213],"adaptive":[215],"ability":[216],"complicated":[218]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":16},{"year":2024,"cited_by_count":10}],"updated_date":"2026-03-04T09:10:02.777135","created_date":"2025-10-10T00:00:00"}