{"id":"https://openalex.org/W4401201162","doi":"https://doi.org/10.48550/arxiv.2407.19094","title":"Wonderful Team: Zero-Shot Physical Task Planning with Visual LLMs","display_name":"Wonderful Team: Zero-Shot Physical Task Planning with Visual LLMs","publication_year":2024,"publication_date":"2024-07-26","ids":{"openalex":"https://openalex.org/W4401201162","doi":"https://doi.org/10.48550/arxiv.2407.19094"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2407.19094","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.19094","pdf_url":"https://arxiv.org/pdf/2407.19094","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.19094","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5066992556","display_name":"Zidan Wang","orcid":"https://orcid.org/0000-0002-4028-2271","observed_orcids":["https://orcid.org/0000-0002-4028-2271"]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]},{"id":"https://openalex.org/I4210100400","display_name":"Northwestern University","ror":"https://ror.org/00m6w7z96","country_code":"PH","type":"education","lineage":["https://openalex.org/I4210100400"]}],"countries":["PH","US"],"is_corresponding":false,"raw_author_name":"Wang, Zidan","raw_affiliation_strings":["Department of Statistics , Northwestern University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Statistics , Northwestern University","institution_ids":["https://openalex.org/I111979921","https://openalex.org/I4210100400"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072983347","display_name":"Rui Shen","orcid":"https://orcid.org/0000-0001-8849-2836","observed_orcids":["https://orcid.org/0000-0001-8849-2836","https://orcid.org/0009-0005-0991-724X","https://orcid.org/0009-0006-8573-0011"]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]},{"id":"https://openalex.org/I4210100400","display_name":"Northwestern University","ror":"https://ror.org/00m6w7z96","country_code":"PH","type":"education","lineage":["https://openalex.org/I4210100400"]},{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["PH","US"],"is_corresponding":false,"raw_author_name":"Shen, Rui","raw_affiliation_strings":["Department of Computer Science , University of Virginia","Department of Statistics , Northwestern University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , University of Virginia","institution_ids":["https://openalex.org/I51556381"]},{"raw_affiliation_string":"Department of Statistics , Northwestern University","institution_ids":["https://openalex.org/I111979921","https://openalex.org/I4210100400"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004762590","display_name":"Bradly C. Stadie","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]},{"id":"https://openalex.org/I4210100400","display_name":"Northwestern University","ror":"https://ror.org/00m6w7z96","country_code":"PH","type":"education","lineage":["https://openalex.org/I4210100400"]}],"countries":["PH","US"],"is_corresponding":false,"raw_author_name":"Stadie, Bradly","raw_affiliation_strings":["Department of Statistics , Northwestern University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Statistics , Northwestern University","institution_ids":["https://openalex.org/I111979921","https://openalex.org/I4210100400"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":14,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9740999937057495,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9740999937057495,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.01899999938905239,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0026000000070780516,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-vision-language-models","display_name":"large vision-language models","score":0.8629999756813049},{"id":"https://openalex.org/keywords/multi-agent-planning","display_name":"multi-agent planning","score":0.8040000200271606},{"id":"https://openalex.org/keywords/robotic-manipulation","display_name":"robotic manipulation","score":0.9049999713897705},{"id":"https://openalex.org/keywords/action-sequence-generation","display_name":"action sequence generation","score":0.6050000190734863},{"id":"https://openalex.org/keywords/semantic-reasoning","display_name":"semantic reasoning","score":0.6700000166893005}],"concepts":[{"id":"https://openalex.org/C2780813799","wikidata":"https://www.wikidata.org/wiki/Q3274237","display_name":"Zero (linguistics)","level":2,"score":0.7449133992195129},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.722241997718811},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6799972653388977},{"id":"https://openalex.org/C2778344882","wikidata":"https://www.wikidata.org/wiki/Q278938","display_name":"Shot (pellet)","level":2,"score":0.6798385381698608},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5099796056747437},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.5069546699523926},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.2349582016468048},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.17500483989715576},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.14295479655265808},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2407.19094","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.19094","pdf_url":"https://arxiv.org/pdf/2407.19094","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.19094","doi":"https://doi.org/10.48550/arxiv.2407.19094","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.19094","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.19094","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.19094","pdf_url":"https://arxiv.org/pdf/2407.19094","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320308380","display_name":"Yale University","ror":"https://ror.org/03v76x132"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4401201162.pdf","grobid_xml":"https://content.openalex.org/works/W4401201162.grobid-xml"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W3030163527","https://openalex.org/W4221152848","https://openalex.org/W4285428875","https://openalex.org/W4323066621","https://openalex.org/W4324128075","https://openalex.org/W4377164392","https://openalex.org/W4384264726","https://openalex.org/W4385430679","https://openalex.org/W4385473486","https://openalex.org/W4396822572","https://openalex.org/W4399374221","https://openalex.org/W4399695759","https://openalex.org/W7133220882"],"related_works":["https://openalex.org/W4389218004","https://openalex.org/W7131402460","https://openalex.org/W7106246472","https://openalex.org/W4404569814","https://openalex.org/W4417251871","https://openalex.org/W4404988990","https://openalex.org/W7138881487","https://openalex.org/W7153089423","https://openalex.org/W4401687172","https://openalex.org/W4390783932"],"abstract_inverted_index":{"We":[0,177],"introduce":[1],"Wonderful":[2,97],"Team,":[3],"a":[4,18,30,35,45,84,95,152,163],"multi-agent":[5],"Vision":[6],"Large":[7],"Language":[8],"Model":[9],"(VLLM)":[10],"framework":[11],"for":[12,29,57,67,71,78,201],"executing":[13],"high-level":[14,25,68,203],"robotic":[15,72,204],"planning":[16,26,70,81,105,205],"in":[17,187,207],"zero-shot":[19,24],"regime.":[20],"In":[21],"our":[22,74],"context,":[23],"means":[27],"that":[28,110],"novel":[31],"environment,":[32],"we":[33,118],"provide":[34],"VLLM":[36,50],"with":[37,173],"an":[38,120,134,155,199],"image":[39],"of":[40,54,166,185],"the":[41,49,52,58,62,79,144,182,188,193,208],"robot's":[42],"surroundings":[43],"and":[44,48,92,103,150,154,191],"task":[46],"description,":[47],"outputs":[51],"sequence":[53],"actions":[55],"necessary":[56],"robot":[59],"to":[60,195],"complete":[61],"task.":[63],"Unlike":[64],"previous":[65],"methods":[66,109,130],"visual":[69],"manipulation,":[73],"method":[75],"uses":[76],"VLLMs":[77,186,197],"entire":[80],"process,":[82],"enabling":[83],"more":[85],"tightly":[86],"integrated":[87],"loop":[88],"between":[89],"perception,":[90],"control,":[91],"planning.":[93],"As":[94],"result,":[96],"Team's":[98],"performance":[99],"on":[100,112,126,141,162],"real-world":[101],"semantic":[102,167],"physical":[104],"tasks":[106,142,169],"often":[107],"exceeds":[108],"rely":[111],"separate":[113],"vision":[114],"systems.":[115],"For":[116],"example,":[117],"see":[119],"average":[121,135,156],"40%":[122],"success":[123],"rate":[124],"improvement":[125,137,158],"VimaBench":[127],"over":[128,138,159],"prior":[129],"such":[131],"as":[132,198],"NLaP,":[133],"30%":[136],"Trajectory":[139,145,160],"Generators":[140,161],"from":[143],"Generator":[146],"paper,":[147],"including":[148,170],"drawing":[149],"wiping":[151],"plate,":[153],"70%":[157],"new":[164],"set":[165],"reasoning":[168],"environment":[171],"rearrangement":[172],"implicit":[174],"linguistic":[175],"constraints.":[176],"hope":[178],"these":[179],"results":[180],"highlight":[181],"rapid":[183],"improvements":[184],"past":[189],"year,":[190],"motivate":[192],"community":[194],"consider":[196],"option":[200],"some":[202],"problems":[206],"future.":[209]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2024-08-01T00:00:00"}