{"id":"https://openalex.org/W7204958905","doi":"https://doi.org/10.21203/rs.3.rs-10643317/v1","title":"Semantic Planning with Large Language Models and Vision-Language Models for Collaborative Robots: A Review","display_name":"Semantic Planning with Large Language Models and Vision-Language Models for Collaborative Robots: A Review","publication_year":2026,"publication_date":"2026-09-01","ids":{"openalex":"https://openalex.org/W7204958905","doi":"https://doi.org/10.21203/rs.3.rs-10643317/v1"},"language":"en","primary_location":{"id":"doi:10.21203/rs.3.rs-10643317/v1","doi":"https://doi.org/10.21203/rs.3.rs-10643317/v1","is_oa":true,"landing_page_url":"https://doi.org/10.21203/rs.3.rs-10643317/v1","pdf_url":"https://www.researchsquare.com/article/rs-10643317/latest.pdf","source":{"id":"https://openalex.org/S4306525896","display_name":"Research Square","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Research Square","raw_type":"posted-content"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.researchsquare.com/article/rs-10643317/latest.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102549428","display_name":"Osama Ali Khan","orcid":"https://orcid.org/0009-0004-8240-3774","observed_orcids":["https://orcid.org/0009-0004-8240-3774"]},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Osama Ali Khan","raw_affiliation_strings":["Southern University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Southern University of Science and Technology","institution_ids":["https://openalex.org/I3045169105"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5148925227","display_name":"Abuzar Ghaffari","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I90610274","display_name":"University of Engineering and Technology Peshawar","ror":"https://ror.org/00p034093","country_code":"PK","type":"education","lineage":["https://openalex.org/I90610274"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Abuzar Ghaffari","raw_affiliation_strings":["University of Engineering and Technology Peshawar"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Engineering and Technology Peshawar","institution_ids":["https://openalex.org/I90610274"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5102549428"],"corresponding_institution_ids":["https://openalex.org/I3045169105"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.004600000102072954,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/semantic-planning","display_name":"semantic planning","score":0.9810000061988831},{"id":"https://openalex.org/keywords/collaborative-robots","display_name":"collaborative robots","score":0.8209999799728394},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8930000066757202},{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":1.0},{"id":"https://openalex.org/keywords/task-and-motion-planning","display_name":"task and motion planning","score":0.7900000214576721},{"id":"https://openalex.org/keywords/human-robot-interaction","display_name":"human-robot interaction","score":0.9200000166893005},{"id":"https://openalex.org/keywords/groundedness","display_name":"groundedness","score":0.6269999742507935},{"id":"https://openalex.org/keywords/planning-under-uncertainty","display_name":"planning under uncertainty","score":0.5580000281333923},{"id":"https://openalex.org/keywords/constraint-violation-detection","display_name":"constraint violation detection","score":0.7089999914169312},{"id":"https://openalex.org/keywords/plan-verification","display_name":"plan verification","score":0.597000002861023},{"id":"https://openalex.org/keywords/uncertainty-estimation","display_name":"uncertainty estimation","score":0.6190000176429749},{"id":"https://openalex.org/keywords/foundation-models","display_name":"foundation models","score":0.3659999966621399}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7336999773979187},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6111000180244446},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.544700026512146},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5394999980926514},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.5252000093460083},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5115000009536743},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.489300012588501},{"id":"https://openalex.org/C2775955345","wikidata":"https://www.wikidata.org/wiki/Q7449071","display_name":"Semantic mapping","level":2,"score":0.4440999925136566},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.44110000133514404},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.41040000319480896},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.36800000071525574},{"id":"https://openalex.org/C81074085","wikidata":"https://www.wikidata.org/wiki/Q366872","display_name":"Motion planning","level":3,"score":0.36800000071525574},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.366100013256073},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.36399999260902405},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.349700003862381},{"id":"https://openalex.org/C90312973","wikidata":"https://www.wikidata.org/wiki/Q7449052","display_name":"Semantic data model","level":2,"score":0.3384000062942505},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.33820000290870667},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.3328999876976013},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.32100000977516174},{"id":"https://openalex.org/C2780069185","wikidata":"https://www.wikidata.org/wiki/Q7977945","display_name":"Equivalence (formal languages)","level":2,"score":0.303600013256073},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3018999993801117},{"id":"https://openalex.org/C198942812","wikidata":"https://www.wikidata.org/wiki/Q496618","display_name":"Semantic property","level":2,"score":0.298799991607666},{"id":"https://openalex.org/C114073186","wikidata":"https://www.wikidata.org/wiki/Q2631895","display_name":"Automated planning and scheduling","level":2,"score":0.29510000348091125},{"id":"https://openalex.org/C69075417","wikidata":"https://www.wikidata.org/wiki/Q515701","display_name":"Linked data","level":3,"score":0.2919999957084656},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2874000072479248},{"id":"https://openalex.org/C17231256","wikidata":"https://www.wikidata.org/wiki/Q5156540","display_name":"Completeness (order theory)","level":2,"score":0.2782000005245209},{"id":"https://openalex.org/C207347870","wikidata":"https://www.wikidata.org/wiki/Q371174","display_name":"Gesture","level":2,"score":0.2727000117301941},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.2590000033378601},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.25839999318122864},{"id":"https://openalex.org/C124681953","wikidata":"https://www.wikidata.org/wiki/Q339062","display_name":"Decomposition","level":2,"score":0.2556999921798706}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.21203/rs.3.rs-10643317/v1","doi":"https://doi.org/10.21203/rs.3.rs-10643317/v1","is_oa":true,"landing_page_url":"https://doi.org/10.21203/rs.3.rs-10643317/v1","pdf_url":"https://www.researchsquare.com/article/rs-10643317/latest.pdf","source":{"id":"https://openalex.org/S4306525896","display_name":"Research Square","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Research Square","raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.21203/rs.3.rs-10643317/v1","doi":"https://doi.org/10.21203/rs.3.rs-10643317/v1","is_oa":true,"landing_page_url":"https://doi.org/10.21203/rs.3.rs-10643317/v1","pdf_url":"https://www.researchsquare.com/article/rs-10643317/latest.pdf","source":{"id":"https://openalex.org/S4306525896","display_name":"Research Square","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Research Square","raw_type":"posted-content"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204958905.pdf","grobid_xml":"https://content.openalex.org/works/W7204958905.grobid-xml"},"referenced_works_count":123,"referenced_works":["https://openalex.org/W46490633","https://openalex.org/W1483019628","https://openalex.org/W1545688112","https://openalex.org/W1992461594","https://openalex.org/W2019160338","https://openalex.org/W2032426551","https://openalex.org/W2044140042","https://openalex.org/W2046213647","https://openalex.org/W2063052894","https://openalex.org/W2070935310","https://openalex.org/W2101876245","https://openalex.org/W2105925198","https://openalex.org/W2107019937","https://openalex.org/W2116139612","https://openalex.org/W2119709400","https://openalex.org/W2135639338","https://openalex.org/W2135907989","https://openalex.org/W2141841102","https://openalex.org/W2143074722","https://openalex.org/W2146948159","https://openalex.org/W2151958719","https://openalex.org/W2157289187","https://openalex.org/W2161414194","https://openalex.org/W2236233024","https://openalex.org/W2280090643","https://openalex.org/W2337392266","https://openalex.org/W2462906003","https://openalex.org/W2470650155","https://openalex.org/W2588978360","https://openalex.org/W2605102758","https://openalex.org/W2621218529","https://openalex.org/W2765218414","https://openalex.org/W2783230551","https://openalex.org/W2784743256","https://openalex.org/W2784881541","https://openalex.org/W2786819761","https://openalex.org/W2789905221","https://openalex.org/W2927819804","https://openalex.org/W2993086250","https://openalex.org/W3003205975","https://openalex.org/W3007769740","https://openalex.org/W3010621561","https://openalex.org/W3034723486","https://openalex.org/W3088310808","https://openalex.org/W3090293438","https://openalex.org/W3093967600","https://openalex.org/W3126325318","https://openalex.org/W3152331843","https://openalex.org/W3188067267","https://openalex.org/W3202183072","https://openalex.org/W4210659164","https://openalex.org/W4224912544","https://openalex.org/W4226073959","https://openalex.org/W4226352076","https://openalex.org/W4229856339","https://openalex.org/W4243342770","https://openalex.org/W4248056173","https://openalex.org/W4285428875","https://openalex.org/W4285429195","https://openalex.org/W4304195432","https://openalex.org/W4309674289","https://openalex.org/W4312424618","https://openalex.org/W4313026212","https://openalex.org/W4323066621","https://openalex.org/W4361012985","https://openalex.org/W4366999541","https://openalex.org/W4378945636","https://openalex.org/W4380993732","https://openalex.org/W4382603221","https://openalex.org/W4383097638","https://openalex.org/W4383108296","https://openalex.org/W4383108457","https://openalex.org/W4383108895","https://openalex.org/W4383472635","https://openalex.org/W4384264726","https://openalex.org/W4384268338","https://openalex.org/W4385430678","https://openalex.org/W4385430679","https://openalex.org/W4385431115","https://openalex.org/W4385473486","https://openalex.org/W4386215566","https://openalex.org/W4387876242","https://openalex.org/W4388720459","https://openalex.org/W4390783932","https://openalex.org/W4390874280","https://openalex.org/W4390874575","https://openalex.org/W4391631244","https://openalex.org/W4393160795","https://openalex.org/W4399695759","https://openalex.org/W4401414715","https://openalex.org/W4401415287","https://openalex.org/W4401415431","https://openalex.org/W4401416363","https://openalex.org/W4401416769","https://openalex.org/W4402353985","https://openalex.org/W4402890475","https://openalex.org/W4404612908","https://openalex.org/W4405786713","https://openalex.org/W4407244916","https://openalex.org/W7115589878","https://openalex.org/W7124252579","https://openalex.org/W7128535203","https://openalex.org/W7128605005","https://openalex.org/W7129019408","https://openalex.org/W7133188479","https://openalex.org/W7133193597","https://openalex.org/W7133196460","https://openalex.org/W7133224126","https://openalex.org/W7133233472","https://openalex.org/W7133237354","https://openalex.org/W7133237860","https://openalex.org/W7136130211","https://openalex.org/W7138248713","https://openalex.org/W7156894990","https://openalex.org/W7163010444","https://openalex.org/W7163137793","https://openalex.org/W7163155583","https://openalex.org/W7163320673","https://openalex.org/W7163706828","https://openalex.org/W7164022596","https://openalex.org/W7164167344","https://openalex.org/W7166681660","https://openalex.org/W7166744536"],"related_works":["https://openalex.org/W7214890194","https://openalex.org/W4415796942","https://openalex.org/W7204897166","https://openalex.org/W4416401444","https://openalex.org/W7196957482","https://openalex.org/W7221781706","https://openalex.org/W7124468336","https://openalex.org/W7221041636","https://openalex.org/W4415257341","https://openalex.org/W7129511803"],"abstract_inverted_index":{"Abstract":[0],"Large":[1],"Language":[2],"Models":[3,7],"(LLMs)":[4],"and":[5,16,36,41,44,70,93,96,118,121,141,144,156,178,181,185,231,240],"Vision-Language":[6],"(VLMs)":[8],"now":[9],"allow":[10],"robots":[11],"to":[12,89,97,99],"translate":[13],"natural-language":[14],"instructions":[15],"visual":[17],"observations":[18],"into":[19,163,242],"executable":[20],"task":[21,73,117,139],"plans.":[22],"Despite":[23],"these":[24],"advances,":[25],"the":[26,33,46,67,72,87,110,161,170,176,216,228],"resulting":[27],"plans":[28],"remain":[29],"incompletely":[30],"grounded":[31],"in":[32,38,45,189],"robot's":[34],"capabilities":[35],"affordances,":[37],"scene":[39],"geometry":[40],"physical":[42],"constraints,":[43],"collaborator's":[47],"changing":[48],"state.":[49],"The":[50],"deficiency":[51],"is":[52],"most":[53],"damaging":[54],"over":[55],"long":[56],"horizons,":[57],"where":[58],"a":[59,130,164,200,232],"local":[60],"planning":[61,91,105,128,177],"or":[62],"execution":[63],"error":[64],"propagates":[65],"through":[66],"remaining":[68],"actions":[69],"compromises":[71],"without":[74],"announcing":[75],"itself.":[76],"Reliable":[77],"collaboration":[78],"therefore":[79],"demands":[80],"not":[81],"only":[82],"higher":[83],"per-step":[84,204],"accuracy":[85,205],"but":[86],"means":[88],"detect":[90],"uncertainty":[92,236],"constraint":[94],"violations":[95],"respond":[98],"them.":[100],"This":[101],"survey":[102],"reviews":[103],"semantic":[104,127],"for":[106,234],"collaborative":[107,126,244],"robotics":[108],"at":[109,172],"interface":[111],"of":[112,166,195],"foundation-model":[113],"reasoning,":[114],"multimodal":[115],"perception,":[116],"motion":[119],"planning,":[120],"human-robot":[122],"interaction.":[123],"We":[124,159,197,225],"formalize":[125],"as":[129],"constrained":[131],"grounding":[132],"problem":[133],"relating":[134],"instruction,":[135],"observation,":[136],"robot":[137],"capability,":[138],"state,":[140,143],"human":[142],"assess":[145],"surveyed":[146],"approaches":[147],"against":[148],"five":[149],"explicit":[150,213],"conditions:":[151],"groundedness,":[152],"feasibility,":[153],"safety,":[154],"legibility,":[155],"calibrated":[157],"abstention.":[158],"organize":[160],"field":[162],"taxonomy":[165],"architectures,":[167],"indexed":[168],"by":[169],"depth":[171],"which":[173,209,219],"semantics":[174],"enter":[175],"control":[179],"hierarchy,":[180],"argue":[182],"that":[183],"capability":[184],"inspectability":[186],"have":[187],"moved":[188],"opposite":[190],"directions":[191],"across":[192],"successive":[193],"generations":[194],"architectures.":[196,245],"then":[198],"state":[199],"reliability":[201],"decomposition":[202],"separating":[203],"from":[206],"detection":[207,221],"quality,":[208],"gives,":[210],"under":[211,218],"an":[212],"independent-step":[214],"model,":[215],"condition":[217],"improving":[220,223],"dominates":[222],"accuracy.":[224],"close":[226],"with":[227],"open":[229],"challenges":[230],"roadmap":[233],"incorporating":[235],"estimation,":[237],"plan":[238],"verification,":[239],"recovery":[241],"next-generation":[243]},"counts_by_year":[],"updated_date":"2026-11-12T12:28:57.059296","created_date":"2026-09-02T00:00:00"}