{"id":"https://openalex.org/W4416259481","doi":"https://doi.org/10.48550/arxiv.2509.18953","title":"Eva-VLA: Evaluating Vision-Language-Action Models' Robustness Under Real-World Physical Variations","display_name":"Eva-VLA: Evaluating Vision-Language-Action Models' Robustness Under Real-World Physical Variations","publication_year":2025,"publication_date":"2025-09-23","ids":{"openalex":"https://openalex.org/W4416259481","doi":"https://doi.org/10.48550/arxiv.2509.18953"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2509.18953","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.18953","pdf_url":"https://arxiv.org/pdf/2509.18953","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2509.18953","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103325940","display_name":"Hanqing Liu","orcid":"https://orcid.org/0000-0002-1295-4851","observed_orcids":["https://orcid.org/0000-0002-1295-4851"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Hanqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ruan, Shouwei","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruan, Shouwei","raw_affiliation_strings":["Institute of Artificial Intelligence , Beihang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Artificial Intelligence , Beihang University","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140371092","display_name":"Jiahuan Long","orcid":"https://orcid.org/0000-0001-6892-4101","observed_orcids":["https://orcid.org/0000-0001-6892-4101"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Long, Jiahuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115604995","display_name":"Junqi Wu","orcid":"https://orcid.org/0009-0005-9533-332X","observed_orcids":["https://orcid.org/0009-0005-9533-332X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Junqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111990376","display_name":"Jiacheng Hou","orcid":"https://orcid.org/0009-0004-7081-2319","observed_orcids":["https://orcid.org/0009-0004-7081-2319"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hou, Jiacheng","raw_affiliation_strings":["Defense Innovation Institute , Chinese Academy of Military Science","Intelligent Game and Decision Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Defense Innovation Institute , Chinese Academy of Military Science","institution_ids":[]},{"raw_affiliation_string":"Intelligent Game and Decision Laboratory","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102013771","display_name":"Huili Tang","orcid":"https://orcid.org/0009-0000-6213-5849","observed_orcids":["https://orcid.org/0009-0000-6213-5849"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Huili","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068914676","display_name":"Tingsong Jiang","orcid":"https://orcid.org/0000-0003-1637-2928","observed_orcids":["https://orcid.org/0000-0003-1637-2928"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Tingsong","raw_affiliation_strings":["Defense Innovation Institute , Chinese Academy of Military Science","Intelligent Game and Decision Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Defense Innovation Institute , Chinese Academy of Military Science","institution_ids":[]},{"raw_affiliation_string":"Intelligent Game and Decision Laboratory","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001113052","display_name":"Weien Zhou","orcid":"https://orcid.org/0000-0001-9833-679X","observed_orcids":["https://orcid.org/0000-0001-9833-679X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Weien","raw_affiliation_strings":["Defense Innovation Institute , Chinese Academy of Military Science","Intelligent Game and Decision Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Defense Innovation Institute , Chinese Academy of Military Science","institution_ids":[]},{"raw_affiliation_string":"Intelligent Game and Decision Laboratory","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5089866078","display_name":"Wen Yao","orcid":"https://orcid.org/0000-0001-5224-9834","observed_orcids":["https://orcid.org/0000-0001-5224-9834","https://orcid.org/0000-0001-6730-1201","https://orcid.org/0009-0007-5260-4737","https://orcid.org/0000-0002-3726-7716"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Wen","raw_affiliation_strings":["Defense Innovation Institute , Chinese Academy of Military Science","Intelligent Game and Decision Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Defense Innovation Institute , Chinese Academy of Military Science","institution_ids":[]},{"raw_affiliation_string":"Intelligent Game and Decision Laboratory","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.004900000058114529,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.0010999999940395355,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vision-language-action-models","display_name":"vision-language-action models","score":0.996999979019165},{"id":"https://openalex.org/keywords/robotic-manipulation","display_name":"robotic manipulation","score":0.8450000286102295},{"id":"https://openalex.org/keywords/illumination-variation","display_name":"illumination variation","score":0.8259999752044678},{"id":"https://openalex.org/keywords/adversarial-training","display_name":"adversarial training","score":0.9589999914169312},{"id":"https://openalex.org/keywords/data-augmentation","display_name":"data augmentation","score":0.8949999809265137},{"id":"https://openalex.org/keywords/openvla","display_name":"OpenVLA","score":0.871999979019165},{"id":"https://openalex.org/keywords/real-world-deployment","display_name":"field deployment","score":0.5120000243186951}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.828499972820282},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.779699981212616},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6826000213623047},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.42399999499320984},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41260001063346863},{"id":"https://openalex.org/C116672817","wikidata":"https://www.wikidata.org/wiki/Q1454986","display_name":"Physical system","level":2,"score":0.3758000135421753},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.35109999775886536},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.3483999967575073},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.32850000262260437},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3255999982357025},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.30720001459121704},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.30709999799728394},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.2921999990940094},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2556000053882599},{"id":"https://openalex.org/C31531917","wikidata":"https://www.wikidata.org/wiki/Q915157","display_name":"Robust control","level":3,"score":0.2508000135421753}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2509.18953","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.18953","pdf_url":"https://arxiv.org/pdf/2509.18953","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2509.18953","doi":"https://doi.org/10.48550/arxiv.2509.18953","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2509.18953","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2509.18953","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.18953","pdf_url":"https://arxiv.org/pdf/2509.18953","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416259481.pdf","grobid_xml":"https://content.openalex.org/works/W4416259481.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4415276858","https://openalex.org/W4411449683","https://openalex.org/W7117174070","https://openalex.org/W7128388565","https://openalex.org/W7216131719","https://openalex.org/W7131819932","https://openalex.org/W4416344002","https://openalex.org/W4415945598","https://openalex.org/W4404216506","https://openalex.org/W7084389655"],"abstract_inverted_index":{"Vision-Language-Action":[0],"(VLA)":[1],"models":[2,39],"have":[3],"emerged":[4],"as":[5,45,217],"promising":[6],"solutions":[7],"for":[8],"robotic":[9,228],"manipulation,":[10],"yet":[11],"their":[12],"robustness":[13,36,60],"to":[14,32,64,80,223],"real-world":[15,72,88,99,209],"physical":[16,43,59,68,172],"variations":[17,44,100,173],"remains":[18],"critically":[19],"underexplored.":[20],"To":[21,92],"bridge":[22],"this":[23,199],"gap,":[24],"we":[25,97,129],"propose":[26],"Eva-VLA,":[27],"the":[28,35,94,126,146,155,175,184,196,204,212,225],"first":[29,95],"unified":[30],"framework":[31,51,214],"systematically":[33,65],"evaluate":[34],"of":[37,149,157,167,198,227],"VLA":[38,57],"by":[40],"formulating":[41],"uncontrollable":[42],"continuous":[46,132,142],"optimization":[47,134],"problems.":[48],"Specifically,":[49],"our":[50,158],"addresses":[52],"two":[53],"fundamental":[54],"challenges":[55],"in":[56,71],"models'":[58],"evaluation:":[61],"1)":[62],"how":[63,79],"characterize":[66],"diverse":[67],"perturbations":[69,139],"encountered":[70],"deployment":[73],"while":[74,211],"maintaining":[75],"reproducibility,":[76],"and":[77,118,208],"2)":[78],"efficiently":[81],"discover":[82],"worst-case":[83,150,186],"scenarios":[84,187],"without":[85],"incurring":[86],"prohibitive":[87],"data":[89,220],"collection":[90],"costs.":[91],"tackle":[93],"challenge,":[96,128],"decouple":[98],"into":[101,140],"three":[102,171],"key":[103],"dimensions:":[104],"3D":[105],"object":[106],"transformations":[107],"that":[108,114,121,136],"affect":[109],"spatial":[110],"reasoning,":[111],"illumination":[112],"changes":[113],"challenge":[115],"visual":[116],"perception,":[117],"adversarial":[119,189],"regions":[120],"disrupt":[122],"scene":[123],"understanding.":[124],"For":[125],"second":[127],"introduce":[130],"a":[131,141],"black-box":[133],"mechanism":[135],"maps":[137],"these":[138],"parameter":[143],"space,":[144],"enabling":[145],"systematic":[147],"exploration":[148],"scenarios.":[151],"Extensive":[152],"experiments":[153],"validate":[154],"effectiveness":[156,197],"approach.":[159,200],"Notably,":[160],"OpenVLA":[161],"exhibits":[162],"an":[163,218],"average":[164],"failure":[165],"rate":[166],"over":[168],"90%":[169],"across":[170],"on":[174],"LIBERO-Long":[176],"task,":[177],"exposing":[178],"critical":[179],"systemic":[180],"fragilities.":[181],"Furthermore,":[182],"applying":[183],"generated":[185],"during":[188],"training":[190],"quantifiably":[191],"increases":[192],"model":[193],"robustness,":[194],"validating":[195],"Our":[201],"evaluation":[202],"exposes":[203],"gap":[205],"between":[206],"laboratory":[207],"conditions,":[210],"Eva-VLA":[213],"can":[215],"serve":[216],"effective":[219],"augmentation":[221],"method":[222],"enhance":[224],"resilience":[226],"manipulation":[229],"systems.":[230]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2025-10-10T00:00:00"}