{"id":"https://openalex.org/W4415312650","doi":"https://doi.org/10.48550/arxiv.2506.14727","title":"Casper: Inferring Diverse Intents for Assistive Teleoperation with Vision Language Models","display_name":"Casper: Inferring Diverse Intents for Assistive Teleoperation with Vision Language Models","publication_year":2025,"publication_date":"2025-06-17","ids":{"openalex":"https://openalex.org/W4415312650","doi":"https://doi.org/10.48550/arxiv.2506.14727"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2506.14727","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.14727","pdf_url":"https://arxiv.org/pdf/2506.14727","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2506.14727","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068139117","display_name":"Huihan Liu","orcid":"https://orcid.org/0009-0001-5931-0287","observed_orcids":["https://orcid.org/0009-0001-5931-0287"]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liu, Huihan","raw_affiliation_strings":["The University of Texas at Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047602471","display_name":"Rutav Shah","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shah, Rutav","raw_affiliation_strings":["The University of Texas at Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052644162","display_name":"Shuijing Liu","orcid":"https://orcid.org/0000-0002-0038-2956","observed_orcids":["https://orcid.org/0000-0002-0038-2956"]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liu, Shuijing","raw_affiliation_strings":["The University of Texas at Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120048441","display_name":"Jack Pittenger","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pittenger, Jack","raw_affiliation_strings":["The University of Texas at Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075135654","display_name":"Mingyo Seo","orcid":"https://orcid.org/0000-0003-0966-0462","observed_orcids":["https://orcid.org/0000-0003-0966-0462"]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Seo, Mingyo","raw_affiliation_strings":["The University of Texas at Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070110088","display_name":"Yuchen Cui","orcid":"https://orcid.org/0000-0001-7417-1222","observed_orcids":["https://orcid.org/0000-0001-7417-1222"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cui, Yuchen","raw_affiliation_strings":["The University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041302228","display_name":"Yonatan Bisk","orcid":"https://orcid.org/0000-0002-2111-9081","observed_orcids":["https://orcid.org/0000-0002-2111-9081"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bisk, Yonatan","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065655761","display_name":"Roberto Mart\u00edn-Mart\u00edn","orcid":"https://orcid.org/0000-0002-9586-2759","observed_orcids":["https://orcid.org/0000-0002-9586-2759"]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mart\u00edn-Mart\u00edn, Roberto","raw_affiliation_strings":["The University of Texas at Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5030826237","display_name":"Yuke Zhu","orcid":"https://orcid.org/0000-0002-9198-2227","observed_orcids":["https://orcid.org/0000-0002-9198-2227"]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhu, Yuke","raw_affiliation_strings":["The University of Texas at Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11687","display_name":"Teleoperation and Haptic Systems","score":0.7451000213623047,"subfield":{"id":"https://openalex.org/subfields/2210","display_name":"Mechanical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11687","display_name":"Teleoperation and Haptic Systems","score":0.7451000213623047,"subfield":{"id":"https://openalex.org/subfields/2210","display_name":"Mechanical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.2395000010728836,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.004399999976158142,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/human-robot-collaboration","display_name":"human-robot collaboration","score":0.847000002861023},{"id":"https://openalex.org/keywords/intent-inference","display_name":"intention inference","score":0.9459999799728394},{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.9330000281333923},{"id":"https://openalex.org/keywords/commonsense-reasoning","display_name":"commonsense reasoning","score":0.925000011920929},{"id":"https://openalex.org/keywords/skill-library","display_name":"skill library","score":0.7590000033378601},{"id":"https://openalex.org/keywords/mobile-manipulation","display_name":"mobile manipulation","score":0.9549999833106995},{"id":"https://openalex.org/keywords/user-satisfaction","display_name":"user satisfaction","score":0.6869999766349792}],"concepts":[{"id":"https://openalex.org/C161759796","wikidata":"https://www.wikidata.org/wiki/Q3982902","display_name":"Teleoperation","level":3,"score":0.9128000140190125},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7157999873161316},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.6690000295639038},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5336999893188477},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.46059998869895935},{"id":"https://openalex.org/C196467688","wikidata":"https://www.wikidata.org/wiki/Q1851985","display_name":"Telerobotics","level":4,"score":0.4465999901294708},{"id":"https://openalex.org/C145460709","wikidata":"https://www.wikidata.org/wiki/Q859951","display_name":"Human\u2013robot interaction","level":3,"score":0.43630000948905945},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.41600000858306885},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.4122999906539917},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.40790000557899475},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39969998598098755},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.3756999969482422},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.35899999737739563},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.32429999113082886},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.31049999594688416},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.29499998688697815},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2912999987602234},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.28529998660087585},{"id":"https://openalex.org/C60692881","wikidata":"https://www.wikidata.org/wiki/Q584529","display_name":"Humanoid robot","level":3,"score":0.2743000090122223},{"id":"https://openalex.org/C193221554","wikidata":"https://www.wikidata.org/wiki/Q5153664","display_name":"Commonsense reasoning","level":2,"score":0.2712000012397766},{"id":"https://openalex.org/C19966478","wikidata":"https://www.wikidata.org/wiki/Q4810574","display_name":"Mobile robot","level":3,"score":0.2689000070095062},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.2676999866962433},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.2556000053882599},{"id":"https://openalex.org/C203479927","wikidata":"https://www.wikidata.org/wiki/Q5165939","display_name":"Controller (irrigation)","level":2,"score":0.25429999828338623}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2506.14727","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.14727","pdf_url":"https://arxiv.org/pdf/2506.14727","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2506.14727","doi":"https://doi.org/10.48550/arxiv.2506.14727","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2506.14727","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2506.14727","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.14727","pdf_url":"https://arxiv.org/pdf/2506.14727","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4256298581","display_name":"EFRI BRAID: Brain-inspired Algorithms for Autonomous Robots (BAAR)","funder_award_id":"2318065","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7021845547","display_name":"CAREER: Intelligent Manipulation in the Real World via Modularity and Abstraction","funder_award_id":"2145283","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320328359","display_name":"Ministry of Science and ICT, South Korea","ror":"https://ror.org/01wpjm123"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320335489","display_name":"Institute for Information and Communications Technology Promotion","ror":"https://ror.org/01g0hqq23"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"},{"id":"https://openalex.org/F4320337395","display_name":"Division of Emerging Frontiers in Research and Innovation","ror":"https://ror.org/0388pet74"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415312650.pdf","grobid_xml":"https://content.openalex.org/works/W4415312650.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Assistive":[0],"teleoperation,":[1],"where":[2],"control":[3,44],"is":[4,30],"shared":[5],"between":[6],"a":[7,10,36,109,117,134],"human":[8,40,157,169],"and":[9,14,20,46,98,115,133,159,172,180],"robot,":[11],"enables":[12],"efficient":[13],"intuitive":[15],"human-robot":[16],"collaboration":[17],"in":[18,26,88],"diverse":[19],"unstructured":[21],"environments.":[22],"A":[23],"central":[24],"challenge":[25],"real-world":[27,74],"assistive":[28,80,143,181],"teleoperation":[29,81,144,179,182],"for":[31,73,94,108],"the":[32,139],"robot":[33],"to":[34,47,58,64,126,146],"infer":[35],"wide":[37],"range":[38],"of":[39,112,129,141],"intentions":[41],"from":[42],"user":[43,131,175],"inputs":[45],"assist":[48],"users":[49],"with":[50],"correct":[51],"actions.":[52],"Existing":[53],"methods":[54],"are":[55],"either":[56],"confined":[57],"simple,":[59],"predefined":[60],"scenarios":[61],"or":[62],"restricted":[63],"task-specific":[65],"data":[66],"distributions":[67],"at":[68],"training,":[69],"limiting":[70],"their":[71],"support":[72,147],"assistance.":[75],"We":[76],"introduce":[77],"Casper,":[78],"an":[79,104],"system":[82,160],"that":[83,122,137,163],"leverages":[84,123],"commonsense":[85,124],"knowledge":[86],"embedded":[87],"pre-trained":[89],"visual":[90],"language":[91],"models":[92],"(VLMs)":[93],"real-time":[95],"intent":[96,119],"inference":[97,120],"flexible":[99],"skill":[100,135],"execution.":[101],"Casper":[102,164],"incorporates":[103],"open-world":[105],"perception":[106],"module":[107],"generalized":[110],"understanding":[111],"novel":[113],"objects":[114],"scenes,":[116],"VLM-powered":[118],"mechanism":[121],"reasoning":[125],"interpret":[127],"snippets":[128],"teleoperated":[130],"input,":[132],"library":[136],"expands":[138],"scope":[140],"prior":[142],"systems":[145],"diverse,":[148],"long-horizon":[149],"mobile":[150],"manipulation":[151],"tasks.":[152],"Extensive":[153],"empirical":[154],"evaluation,":[155],"including":[156],"studies":[158],"ablations,":[161],"demonstrates":[162],"improves":[165],"task":[166],"performance,":[167],"reduces":[168],"cognitive":[170],"load,":[171],"achieves":[173],"higher":[174],"satisfaction":[176],"than":[177],"direct":[178],"baselines.":[183]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-09-30T09:27:27.532063","created_date":"2025-10-18T00:00:00"}