{"id":"https://openalex.org/W7164939736","doi":"https://doi.org/10.48550/arxiv.2606.17043","title":"Hierarchical Advantage Weighting for Online RL Fine-Tuning of VLAs from Sparse Episode Outcomes","display_name":"Hierarchical Advantage Weighting for Online RL Fine-Tuning of VLAs from Sparse Episode Outcomes","publication_year":2026,"publication_date":"2026-06-15","ids":{"openalex":"https://openalex.org/W7164939736","doi":"https://doi.org/10.48550/arxiv.2606.17043"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.17043","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.17043","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.17043","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138693635","display_name":"Tongyan Fang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fang, Tongyan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043900893","display_name":"Sicong Huang","orcid":"https://orcid.org/0000-0003-0854-4734"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Siyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083747119","display_name":"Naiyu Fang","orcid":"https://orcid.org/0000-0003-0145-1690"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fang, Naiyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082683966","display_name":"Ganlong Zhao","orcid":"https://orcid.org/0000-0002-1612-3641"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Ganlong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068399946","display_name":"Zhongjin Luo","orcid":"https://orcid.org/0000-0002-3483-4236"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Zhongjin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138728083","display_name":"Jianbo Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jianbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138698406","display_name":"Xiaogang Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xiaogang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136466952","display_name":"Y H Dong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Ying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138707653","display_name":"Hongsheng Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Hongsheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.7955999970436096,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.7955999970436096,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.03629999980330467,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.030300000682473183,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/weighting","display_name":"Weighting","score":0.6930000185966492},{"id":"https://openalex.org/keywords/outcome","display_name":"Outcome (game theory)","score":0.6355999708175659},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.49399998784065247},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.3950999975204468},{"id":"https://openalex.org/keywords/a-weighting","display_name":"A-weighting","score":0.35929998755455017},{"id":"https://openalex.org/keywords/intervention","display_name":"Intervention (counseling)","score":0.32359999418258667}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7171000242233276},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.6930000185966492},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.6355999708175659},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5148000121116638},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.49399998784065247},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4271000027656555},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3950999975204468},{"id":"https://openalex.org/C70136482","wikidata":"https://www.wikidata.org/wiki/Q13583781","display_name":"A-weighting","level":3,"score":0.35929998755455017},{"id":"https://openalex.org/C2780665704","wikidata":"https://www.wikidata.org/wiki/Q959298","display_name":"Intervention (counseling)","level":2,"score":0.32359999418258667},{"id":"https://openalex.org/C57691317","wikidata":"https://www.wikidata.org/wiki/Q1289248","display_name":"Scalar (mathematics)","level":2,"score":0.3156999945640564},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.305400013923645},{"id":"https://openalex.org/C66905080","wikidata":"https://www.wikidata.org/wiki/Q17005494","display_name":"Binary classification","level":3,"score":0.3043000102043152},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.2928999960422516},{"id":"https://openalex.org/C2778023277","wikidata":"https://www.wikidata.org/wiki/Q321703","display_name":"Premise","level":2,"score":0.26499998569488525},{"id":"https://openalex.org/C75553542","wikidata":"https://www.wikidata.org/wiki/Q178161","display_name":"A priori and a posteriori","level":2,"score":0.2572999894618988}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.17043","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.17043","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.17043","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.17043","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"When":[0],"pretrained":[1],"VLA":[2],"policies":[3],"are":[4],"fine-tuned":[5],"through":[6],"online":[7],"RL,":[8],"each":[9],"rollout":[10],"episode":[11,101],"produces":[12],"only":[13,160],"a":[14,36,61,139],"single":[15,37,62],"binary":[16,79],"outcome":[17,34,181],"(success":[18],"or":[19,40],"failure),":[20],"yet":[21],"the":[22,66,78,167,173,187],"actor":[23,174],"update":[24],"requires":[25],"per-transition":[26,170],"supervision.":[27],"Existing":[28],"approaches":[29],"commonly":[30],"reduce":[31],"this":[32],"sparse":[33],"to":[35,84,158,183,218],"scalar":[38,63],"reward":[39],"advantage":[41],"signal,":[42],"which":[43,121],"conflates":[44,65],"distinct":[45],"forms":[46],"of":[47,69,213],"transition-level":[48],"feedback":[49],"and":[50,71,96,134,156,165,216,221],"provides":[51,81],"limited":[52],"guidance":[53],"once":[54,73],"basic":[55,74],"task":[56],"success":[57,75,153,207],"becomes":[58],"achievable.":[59],"First,":[60],"signal":[64],"two":[67,128],"objectives":[68,129],"viability":[70,151,162],"efficiency;":[72],"is":[76,154,163],"achieved,":[77],"label":[80],"no":[82],"gradient":[83],"distinguish":[85],"efficient":[86],"completions":[87],"from":[88,192,208],"slow":[89],"ones.":[90],"Second,":[91],"real-world":[92],"rollouts":[93],"mix":[94],"autonomous":[95],"intervention":[97,195],"segments;":[98],"naively":[99],"assigning":[100],"outcomes":[102],"across":[103,194],"these":[104,112,127],"boundaries":[105],"introduces":[106],"incorrect":[107],"credit":[108,177],"assignment.":[109],"To":[110],"address":[111],"issues,":[113],"we":[114],"propose":[115],"Hierarchical":[116],"Advantage-Weighted":[117],"Behavior":[118],"Cloning":[119],"(HABC),":[120],"trains":[122],"separate":[123],"critic":[124],"heads":[125],"for":[126],"on":[130,172,200],"different":[131],"data":[132],"subsets":[133],"combines":[135],"their":[136,147],"outputs":[137],"with":[138],"state-adaptive":[140,143],"balance.":[141],"A":[142],"gate":[144],"$g_t$":[145],"merges":[146],"one-step":[148],"advantages,":[149],"prioritizing":[150],"when":[152,161],"uncertain":[155],"shifting":[157],"efficiency":[159],"high,":[164],"converts":[166],"result":[168],"into":[169],"weights":[171],"loss.":[175],"Intervention-aware":[176],"assignment":[178],"further":[179],"restricts":[180],"labels":[182],"segments":[184],"executed":[185],"by":[186],"current":[188],"policy,":[189],"preventing":[190],"supervision":[191],"leaking":[193],"boundaries.":[196],"In":[197],"real-robot":[198],"experiments":[199],"three":[201],"contact-rich":[202],"bimanual":[203],"tasks,":[204],"HABC":[205],"raises":[206],"supervised":[209],"fine-tuning":[210],"(SFT)":[211],"baselines":[212],"36%,":[214],"44%,":[215],"12%":[217],"92%,":[219],"88%,":[220],"38%.":[222]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-17T00:00:00"}