{"id":"https://openalex.org/W4401417048","doi":"https://doi.org/10.1109/icra57147.2024.10610090","title":"Physically Grounded Vision-Language Models for Robotic Manipulation","display_name":"Physically Grounded Vision-Language Models for Robotic Manipulation","publication_year":2024,"publication_date":"2024-05-13","ids":{"openalex":"https://openalex.org/W4401417048","doi":"https://doi.org/10.1109/icra57147.2024.10610090"},"language":"en","primary_location":{"id":"doi:10.1109/icra57147.2024.10610090","doi":"https://doi.org/10.1109/icra57147.2024.10610090","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra57147.2024.10610090","pdf_url":null,"source":{"id":"https://openalex.org/S4210217939","display_name":"Proceedings - IEEE International Conference on Robotics and Automation/Proceedings","issn_l":"1050-4729","issn":["1050-4729","2577-087X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-1"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2309.02561","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016538750","display_name":"Jensen Gao","orcid":"https://orcid.org/0009-0008-8199-6087","observed_orcids":["https://orcid.org/0009-0008-8199-6087"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jensen Gao","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039641037","display_name":"Bidipta Sarkar","orcid":"https://orcid.org/0000-0002-0584-3504","observed_orcids":["https://orcid.org/0000-0002-0584-3504"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bidipta Sarkar","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100676785","display_name":"Fei Xia","orcid":"https://orcid.org/0000-0003-4343-1444","observed_orcids":["https://orcid.org/0000-0003-4343-1444"]},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Fei Xia","raw_affiliation_strings":["Google DeepMind"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google DeepMind","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025016495","display_name":"Ted G. Xiao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ted Xiao","raw_affiliation_strings":["Google DeepMind"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google DeepMind","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100621605","display_name":"Jiajun Wu","orcid":"https://orcid.org/0000-0002-4176-343X","observed_orcids":["https://orcid.org/0000-0002-4176-343X"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiajun Wu","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018507768","display_name":"Brian Ichter","orcid":"https://orcid.org/0000-0002-6955-6432","observed_orcids":["https://orcid.org/0000-0002-6955-6432"]},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Brian Ichter","raw_affiliation_strings":["Google DeepMind"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google DeepMind","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102792178","display_name":"Anirudha Majumdar","orcid":"https://orcid.org/0009-0002-2296-7485","observed_orcids":["https://orcid.org/0009-0002-2296-7485","https://orcid.org/0000-0002-9383-6071"]},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Anirudha Majumdar","raw_affiliation_strings":["Google DeepMind"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google DeepMind","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080725225","display_name":"Dorsa Sadigh","orcid":"https://orcid.org/0000-0002-7802-9183","observed_orcids":["https://orcid.org/0000-0002-7802-9183"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dorsa Sadigh","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":91.8581,"has_fulltext":true,"cited_by_count":179,"citation_normalized_percentile":{"value":0.99971082,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"12462","last_page":"12469"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.7035999894142151,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.7035999894142151,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.29440000653266907,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.9750000238418579},{"id":"https://openalex.org/keywords/robotic-manipulation","display_name":"robotic manipulation","score":0.7870000004768372},{"id":"https://openalex.org/keywords/physical-reasoning","display_name":"physical reasoning","score":0.671999990940094},{"id":"https://openalex.org/keywords/household-objects","display_name":"household products","score":0.6930000185966492},{"id":"https://openalex.org/keywords/interactive-planning","display_name":"interactive planning","score":0.5320000052452087},{"id":"https://openalex.org/keywords/physical-grounding","display_name":"physical grounding","score":0.3330000042915344}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7024863958358765},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5190559029579163},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4797850251197815},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.468959242105484},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4188283085823059}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/icra57147.2024.10610090","doi":"https://doi.org/10.1109/icra57147.2024.10610090","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra57147.2024.10610090","pdf_url":null,"source":{"id":"https://openalex.org/S4210217939","display_name":"Proceedings - IEEE International Conference on Robotics and Automation/Proceedings","issn_l":"1050-4729","issn":["1050-4729","2577-087X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-1"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2309.02561","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.02561","pdf_url":"https://arxiv.org/pdf/2309.02561","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2309.02561","doi":"https://doi.org/10.48550/arxiv.2309.02561","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2309.02561","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2309.02561","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.02561","pdf_url":"https://arxiv.org/pdf/2309.02561","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1590227012","display_name":null,"funder_award_id":"N00014-23-1-2355","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G3928597213","display_name":"CAREER: Physical Object Modeling for Intelligent Systems","funder_award_id":"2338203","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6022427645","display_name":"CAREER: Safe and Influencing Interactions for Human-Robot Systems","funder_award_id":"1941722","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8462687001","display_name":"NRI/Collaborative Research: Robot-Assisted Feeding: Towards Efficient, Safe, and Personalized Caregiving Robots","funder_award_id":"2132847","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4401417048.pdf","grobid_xml":"https://content.openalex.org/works/W4401417048.grobid-xml"},"referenced_works_count":45,"referenced_works":["https://openalex.org/W1583837637","https://openalex.org/W2013784666","https://openalex.org/W2181623680","https://openalex.org/W2277195237","https://openalex.org/W2520762063","https://openalex.org/W2560730294","https://openalex.org/W2612690371","https://openalex.org/W2613576910","https://openalex.org/W2626804490","https://openalex.org/W2735318784","https://openalex.org/W2907424612","https://openalex.org/W2947312908","https://openalex.org/W2963294969","https://openalex.org/W3021925415","https://openalex.org/W3135367836","https://openalex.org/W3171236259","https://openalex.org/W3198002980","https://openalex.org/W3199693760","https://openalex.org/W4221152848","https://openalex.org/W4224912544","https://openalex.org/W4225323055","https://openalex.org/W4226278401","https://openalex.org/W4226352076","https://openalex.org/W4280647773","https://openalex.org/W4285428875","https://openalex.org/W4296406182","https://openalex.org/W4305028650","https://openalex.org/W4322716427","https://openalex.org/W4323572061","https://openalex.org/W4324299512","https://openalex.org/W4324321291","https://openalex.org/W4376312115","https://openalex.org/W4383108895","https://openalex.org/W4385565405","https://openalex.org/W4386065596","https://openalex.org/W4386065742","https://openalex.org/W4386071509","https://openalex.org/W4388720459","https://openalex.org/W4389520366","https://openalex.org/W4389665575","https://openalex.org/W6849177959","https://openalex.org/W6849783008","https://openalex.org/W7133211372","https://openalex.org/W7133224126","https://openalex.org/W7133244567"],"related_works":["https://openalex.org/W4417144670","https://openalex.org/W7138881487","https://openalex.org/W4416519712","https://openalex.org/W4415198452","https://openalex.org/W4414580736","https://openalex.org/W4399598100","https://openalex.org/W7158535873","https://openalex.org/W4413146494","https://openalex.org/W4405786114","https://openalex.org/W4413146350"],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2,45,134],"vision-language":[3],"models":[4,23],"(VLMs)":[5],"have":[6],"led":[7],"to":[8,27,115,160],"improved":[9,147],"performance":[10,149],"on":[11,104,150,179],"tasks":[12,65,151],"such":[13,36,73],"as":[14,37],"visual":[15,126],"question":[16],"answering":[17],"and":[18,69,88,145,193,197],"image":[19],"captioning.":[20],"Consequently,":[21],"these":[22,123],"are":[24,43],"now":[25],"well-positioned":[26],"reason":[28],"about":[29,72,155],"the":[30,49,172],"physical":[31,50,70,91,110,156],"world,":[32],"particularly":[33],"within":[34],"domains":[35],"robotic":[38,63,143],"manipulation.":[39],"However,":[40],"current":[41],"VLMs":[42],"limited":[44],"their":[46,60],"understanding":[47,108],"of":[48,55,85,94,109,122,174,199],"concepts":[51,124],"(e.g.,":[52],"material,":[53],"fragility)":[54],"common":[56,95],"objects,":[57],"which":[58],"restricts":[59],"usefulness":[61],"for":[62],"manipulation":[64],"that":[66,100,152,162],"involve":[67],"interaction":[68],"reasoning":[71,154],"objects.":[74,97],"To":[75],"address":[76],"this":[77,130],"limitation,":[78],"we":[79],"propose":[80],"PHYSOBJECTS,":[81],"an":[82,135],"object-centric":[83],"dataset":[84,192],"39.6K":[86],"crowd-sourced":[87],"417K":[89],"automated":[90],"concept":[92],"annotations":[93],"household":[96],"We":[98,128,169,189],"demonstrate":[99],"fine-tuning":[101],"a":[102,139,180],"VLM":[103,133,178],"PhysObjects":[105],"improves":[106,185],"its":[107],"object":[111,157],"concepts,":[112,117,158],"including":[113],"generalization":[114],"held-out":[116],"by":[118],"capturing":[119],"human":[120],"priors":[121],"from":[125],"appearance.":[127],"incorporate":[129],"physically":[131,166,176],"grounded":[132,167,177],"interactive":[136],"framework":[137],"with":[138],"large":[140],"language":[141],"model-based":[142],"planner,":[144],"show":[146],"planning":[148],"require":[153],"compared":[159],"baselines":[161],"do":[163],"not":[164],"leverage":[165],"VLMs.":[168],"additionally":[170],"illustrate":[171],"benefits":[173],"our":[175,191,200],"real":[181],"robot,":[182],"where":[183],"it":[184],"task":[186],"success":[187],"rates.":[188],"release":[190],"provide":[194],"further":[195],"details":[196],"visualizations":[198],"results":[201],"at":[202],"https://iliad.stanford.edu/pg-vlm/.":[203]},"counts_by_year":[{"year":2026,"cited_by_count":54},{"year":2025,"cited_by_count":77},{"year":2024,"cited_by_count":43},{"year":2023,"cited_by_count":5}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-08-09T00:00:00"}