{"id":"https://openalex.org/W7208735440","doi":"https://doi.org/10.48550/arxiv.2609.03276","title":"R2S-Eval: Robot Evaluation with Real-to-Sim Calibration via Vision-Language Models","display_name":"R2S-Eval: Robot Evaluation with Real-to-Sim Calibration via Vision-Language Models","publication_year":2026,"publication_date":"2026-09-03","ids":{"openalex":"https://openalex.org/W7208735440","doi":"https://doi.org/10.48550/arxiv.2609.03276"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.03276","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.03276","pdf_url":"https://arxiv.org/pdf/2609.03276","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.03276","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5149980261","display_name":"Yidi Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang, Yidi","raw_affiliation_strings":["Nanjing Univer- sity ;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanjing Univer- sity ;","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058956860","display_name":"Feixiang Ruan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I116953780","display_name":"Tongji University","ror":"https://ror.org/03rc6as71","country_code":"CN","type":"education","lineage":["https://openalex.org/I116953780"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruan, Feixiang","raw_affiliation_strings":["Tongji University ;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tongji University ;","institution_ids":["https://openalex.org/I116953780"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5150016560","display_name":"Ruoqu Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen, Ruoqu","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148161980","display_name":"Jie Yin","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Jie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5149992760","display_name":"Yang Yu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu, Yang","raw_affiliation_strings":["Nanjing Univer- sity ;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanjing Univer- sity ;","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102020639","display_name":"Mengdi Xu","orcid":"https://orcid.org/0000-0001-6087-7558","observed_orcids":["https://orcid.org/0000-0001-6087-7558"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xu, Mengdi","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5150008344","display_name":"Kaifeng Zhang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Kaifeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9556000232696533,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9556000232696533,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.04050000011920929,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.8050000071525574},{"id":"https://openalex.org/keywords/simulator-calibration","display_name":"simulator calibration","score":0.5950000286102295},{"id":"https://openalex.org/keywords/policy-ranking","display_name":"policy ranking","score":0.5320000052452087},{"id":"https://openalex.org/keywords/human-preference-alignment","display_name":"human preference alignment","score":0.5809999704360962}],"concepts":[{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.7293999791145325},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.7239999771118164},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.7196000218391418},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7017999887466431},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.5791000127792358},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5367000102996826},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.5230000019073486},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41690000891685486},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4138000011444092},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.40209999680519104},{"id":"https://openalex.org/C150415221","wikidata":"https://www.wikidata.org/wiki/Q40687","display_name":"Robotic arm","level":2,"score":0.3885999917984009},{"id":"https://openalex.org/C16345878","wikidata":"https://www.wikidata.org/wiki/Q107472979","display_name":"Orientation (vector space)","level":2,"score":0.3765999972820282},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.3747999966144562},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.35850000381469727},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.3449999988079071},{"id":"https://openalex.org/C44154836","wikidata":"https://www.wikidata.org/wiki/Q45045","display_name":"Simulation","level":1,"score":0.3343999981880188},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.33009999990463257},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.31690001487731934},{"id":"https://openalex.org/C17020691","wikidata":"https://www.wikidata.org/wiki/Q139677","display_name":"Operator (biology)","level":5,"score":0.28630000352859497},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2705000042915344},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.260699987411499},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2578999996185303},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2554999887943268}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.03276","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.03276","pdf_url":"https://arxiv.org/pdf/2609.03276","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.03276","doi":"https://doi.org/10.48550/arxiv.2609.03276","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.03276","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.03276","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.03276","pdf_url":"https://arxiv.org/pdf/2609.03276","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7208735440.pdf","grobid_xml":"https://content.openalex.org/works/W7208735440.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7125820564","https://openalex.org/W4414581984","https://openalex.org/W7101785451","https://openalex.org/W4417055114","https://openalex.org/W7164546197","https://openalex.org/W7147028917","https://openalex.org/W4416878674","https://openalex.org/W7169572231","https://openalex.org/W7214220112","https://openalex.org/W7116303404"],"abstract_inverted_index":{"Evaluating":[0],"robot":[1,67,214,228],"manipulation":[2],"policies":[3],"is":[4],"becoming":[5],"increasingly":[6],"important":[7],"as":[8],"generalist":[9],"models,":[10,14],"particularly":[11],"vision-language-action":[12],"(VLA)":[13],"are":[15,141,203],"deployed":[16],"on":[17,53,79],"physical":[18],"robots.":[19],"However,":[20],"conventional":[21,169],"real-world":[22,115,170,177],"evaluation":[23,90,116,157,215,226],"remains":[24],"labor-intensive,":[25],"unstable,":[26],"and":[27,38,50,71,136,176,184,198,224],"insufficiently":[28],"informative.":[29],"It":[30],"requires":[31],"repeated":[32,48,123,195],"hardware":[33,124],"trials,":[34],"manual":[35,217],"scene":[36],"resets,":[37],"continuous":[39],"operator":[40],"monitoring,":[41],"may":[42],"produce":[43],"different":[44],"policy":[45,145,161,186],"rankings":[46],"across":[47],"evaluations,":[49],"primarily":[51],"relies":[52],"success-rate":[54],"metrics":[55],"that":[56,92,180,202],"provide":[57],"limited":[58],"information":[59],"about":[60],"execution":[61,131],"quality.":[62],"In":[63,210],"contrast,":[64],"humans":[65],"assess":[66,153],"performance":[68],"by":[69,206],"observing":[70],"comparing":[72],"complete":[73],"behaviors":[74],"rather":[75],"than":[76],"relying":[77],"solely":[78],"binary":[80,207],"success":[81,208,218],"outcomes.":[82],"To":[83],"this":[84],"end,":[85],"we":[86],"propose":[87],"R2S-Eval,":[88],"an":[89],"pipeline":[91,158],"combines":[93],"real-to-sim":[94,103],"calibration":[95],"with":[96,190],"vision-language":[97],"model":[98],"(VLM)":[99],"preference":[100],"evaluation.":[101,171],"The":[102,126],"component":[104],"efficiently":[105],"generates":[106],"rollout":[107,134],"videos":[108,135],"in":[109,173],"a":[110,150],"simulator":[111],"calibrated":[112],"to":[113,152],"the":[114,120,130,155,165],"setting,":[117],"thereby":[118],"reducing":[119],"need":[121],"for":[122],"trials.":[125],"VLM":[127],"evaluator":[128],"assesses":[129],"quality":[132],"of":[133,168,227],"produces":[137,182],"pairwise":[138],"preferences,":[139,192],"which":[140],"subsequently":[142],"aggregated":[143],"into":[144],"rankings.":[146],"We":[147],"further":[148],"introduce":[149],"protocol":[151],"whether":[154],"proposed":[156],"yields":[159],"validated":[160],"conclusions":[162],"while":[163],"mitigating":[164],"key":[166],"challenges":[167],"Experiments":[172],"both":[174],"simulation":[175],"settings":[178],"demonstrate":[179],"R2S-Eval":[181,212],"reliable":[183],"stable":[185],"conclusions,":[187],"achieves":[188],"agreement":[189],"human":[191],"substantially":[193],"reduces":[194],"hardware-operation":[196],"effort,":[197],"reveals":[199],"behavior-quality":[200],"differences":[201],"not":[204],"captured":[205],"labels.":[209],"general,":[211],"advances":[213],"from":[216],"counting":[219],"toward":[220],"automated,":[221],"statistically":[222],"stable,":[223],"quality-aware":[225],"behavior.":[229],"Project":[230],"page:":[231],"https://r2s-eval.github.io.":[232]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-05T00:00:00"}