{"id":"https://openalex.org/W7164336854","doi":"https://doi.org/10.48550/arxiv.2606.12299","title":"Learning What to Say to Your VLA: Mostly Harmless Vision Language Action Model Steering","display_name":"Learning What to Say to Your VLA: Mostly Harmless Vision Language Action Model Steering","publication_year":2026,"publication_date":"2026-06-10","ids":{"openalex":"https://openalex.org/W7164336854","doi":"https://doi.org/10.48550/arxiv.2606.12299"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.12299","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12299","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.12299","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138481926","display_name":"Hyun Joe Jeong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jeong, Hyun Joe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138415099","display_name":"Gokul Swamy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Swamy, Gokul","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137641018","display_name":"Andrea Bajcsy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bajcsy, Andrea","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9004999995231628,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9004999995231628,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.02539999969303608,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.008500000461935997,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.739300012588501},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.623199999332428},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.5807999968528748},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.519599974155426},{"id":"https://openalex.org/keywords/language-understanding","display_name":"Language understanding","score":0.5103999972343445},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.4560999870300293},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"Natural language understanding","score":0.3682999908924103},{"id":"https://openalex.org/keywords/simulation-language","display_name":"Simulation language","score":0.3578000068664551}],"concepts":[{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.739300012588501},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6960999965667725},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.623199999332428},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5807999968528748},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.519599974155426},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.5103999972343445},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4900999963283539},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4706000089645386},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.4560999870300293},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4185999929904938},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.3682999908924103},{"id":"https://openalex.org/C141218545","wikidata":"https://www.wikidata.org/wiki/Q7521336","display_name":"Simulation language","level":2,"score":0.3578000068664551},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3513000011444092},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.3208000063896179},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.31540000438690186},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.30550000071525574},{"id":"https://openalex.org/C39608478","wikidata":"https://www.wikidata.org/wiki/Q5015979","display_name":"Cache language model","level":5,"score":0.3019999861717224},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.2989000082015991},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.2815000116825104},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.26589998602867126},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.2572999894618988}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.12299","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12299","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.12299","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12299","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Vision-Language-Action":[0],"(VLA)":[1],"models":[2,50],"provide":[3],"a":[4,42,66,84],"natural":[5],"language":[6,15,49,72,86,98],"interface":[7],"to":[8,16,53,108,120,139],"robot":[9],"control,":[10],"but":[11],"the":[12,114,121,140,147],"mapping":[13],"from":[14],"behavior":[17],"is":[18],"often":[19],"brittle":[20],"and":[21,47,90,164,170,180],"unintuitive:":[22],"semantically":[23],"similar":[24],"instructions":[25,46],"can":[26,51],"induce":[27],"drastically":[28],"different":[29],"behaviors,":[30],"while":[31],"some":[32],"capabilities":[33],"may":[34],"not":[35,184],"be":[36],"elicitable":[37],"through":[38],"prompting":[39],"alone.":[40],"As":[41],"result,":[43],"both":[44],"human":[45],"zero-shot":[48],"fail":[52],"reliably":[54],"steer":[55],"VLAs":[56],"toward":[57],"successful":[58],"task":[59,78,117],"execution.":[60],"In":[61],"this":[62,105],"work,":[63],"we":[64],"propose":[65],"framework":[67],"that":[68,74,95],"interactively":[69],"searches":[70],"for":[71],"sequences":[73,82],"improve":[75,101],"closed-loop":[76],"VLA":[77,158],"performance,":[79],"distills":[80],"these":[81],"into":[83],"test-time":[85],"feedback":[87],"policy":[88],"(LFP),":[89],"learns":[91],"an":[92],"improvement":[93,106],"head":[94,107],"predicts":[96],"when":[97],"steering":[99,111],"will":[100],"performance.":[102],"We":[103],"conformalize":[104],"prevent":[109],"harmful":[110],"interventions,":[112],"where":[113],"LFP":[115,155,175],"decreases":[116],"performance":[118,159],"relative":[119],"original":[122,141],"instruction":[123],"on":[124,131],"out-of-distribution":[125],"scenarios.":[126],"Crucially,":[127],"our":[128,153,173],"approach":[129],"operates":[130],"arbitrary":[132],"frozen":[133],"pre-trained":[134],"VLAs,":[135],"requiring":[136],"neither":[137],"access":[138],"training":[142],"distribution":[143],"nor":[144],"fine-tuning":[145],"of":[146],"underlying":[148],"model.":[149],"On":[150,168],"seen":[151],"environments,":[152],"conformalized":[154,174],"improves":[156],"base":[157],"by":[160],"24.7%":[161],"in":[162,166],"simulation":[163],"65.0%":[165],"hardware.":[167],"visual":[169],"semantic":[171],"perturbations,":[172],"has":[176],"strong":[177],"harmlessness":[178],"guarantees,":[179],"produces":[181],"recovery":[182],"behaviors":[183],"observed":[185],"with":[186],"open-loop":[187],"prompting.":[188]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-12T00:00:00"}