{"id":"https://openalex.org/W7147234844","doi":"https://doi.org/10.48550/arxiv.2603.27482","title":"Difference Feedback: Generating Multimodal Process-Level Supervision for VLM Reinforcement Learning","display_name":"Difference Feedback: Generating Multimodal Process-Level Supervision for VLM Reinforcement Learning","publication_year":2026,"publication_date":"2026-03-29","ids":{"openalex":"https://openalex.org/W7147234844","doi":"https://doi.org/10.48550/arxiv.2603.27482"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.27482","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.27482","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.27482","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5132567872","display_name":"Feiding","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feiding","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132617829","display_name":"Yongkang Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yongkang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132726257","display_name":"Yuhao Liao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liao, Yuhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008533914","display_name":"Zijian Zeng","orcid":"https://orcid.org/0000-0002-6038-5155"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Zijian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104155847","display_name":"Chunzheng Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Chunzheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065943384","display_name":"Yaozong Zheng","orcid":"https://orcid.org/0009-0007-2664-0574"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Yaozong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132720845","display_name":"Yafei Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yafei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037533280","display_name":"Yeling Peng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Yeling","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036826751","display_name":"Youwei Wang","orcid":"https://orcid.org/0000-0002-5033-5576"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Youwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132603922","display_name":"Sibo Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Sibo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yang, Huiming","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Huiming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121610325","display_name":"Linglin Liao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liao, Linglin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5072540443","display_name":"Shunzhi Yang","orcid":"https://orcid.org/0000-0003-0621-2525"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Shunzhi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9375,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9375,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.011099999770522118,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.009499999694526196,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6705999970436096},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5730000138282776},{"id":"https://openalex.org/keywords/outcome","display_name":"Outcome (game theory)","score":0.5126000046730042},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5026999711990356},{"id":"https://openalex.org/keywords/linkage","display_name":"Linkage (software)","score":0.47360000014305115},{"id":"https://openalex.org/keywords/differential","display_name":"Differential (mechanical device)","score":0.46369999647140503},{"id":"https://openalex.org/keywords/group","display_name":"Group (periodic table)","score":0.34769999980926514}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6990000009536743},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6705999970436096},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6388999819755554},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5730000138282776},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.5126000046730042},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5026999711990356},{"id":"https://openalex.org/C31266012","wikidata":"https://www.wikidata.org/wiki/Q6554340","display_name":"Linkage (software)","level":3,"score":0.47360000014305115},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4641000032424927},{"id":"https://openalex.org/C93226319","wikidata":"https://www.wikidata.org/wiki/Q193137","display_name":"Differential (mechanical device)","level":2,"score":0.46369999647140503},{"id":"https://openalex.org/C2781311116","wikidata":"https://www.wikidata.org/wiki/Q83306","display_name":"Group (periodic table)","level":2,"score":0.34769999980926514},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3237999975681305},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.32170000672340393},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.31790000200271606},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3172999918460846},{"id":"https://openalex.org/C2986089797","wikidata":"https://www.wikidata.org/wiki/Q6501338","display_name":"Visual attention","level":3,"score":0.30820000171661377},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.2745000123977661},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.26409998536109924},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.263700008392334},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.2621000111103058}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.27482","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.27482","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.27482","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.27482","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Vision--language":[0],"models":[1],"(VLMs)":[2],"are":[3],"increasingly":[4],"aligned":[5],"via":[6],"Group":[7],"Relative":[8],"Policy":[9],"Optimization":[10],"(GRPO)-style":[11],"training.":[12],"However,":[13],"relying":[14],"solely":[15],"on":[16,89],"terminal":[17],"outcome":[18],"rewards":[19],"yields":[20],"sparse":[21],"credit":[22],"assignment":[23],"in":[24],"multi-step":[25],"reasoning,":[26],"weakening":[27],"the":[28,61],"linkage":[29],"between":[30],"visual":[31,42,77],"evidence":[32],"and":[33,36,41,79,95],"intermediate":[34],"steps":[35],"often":[37],"causing":[38],"unstable":[39],"optimization":[40],"hallucinations.":[43],"We":[44],"propose":[45],"Differential":[46],"Feedback,":[47],"which":[48],"automatically":[49],"constructs":[50],"token/step-level":[51],"supervision":[52],"masks":[53],"by":[54],"repairing":[55],"erroneous":[56],"reasoning":[57,91],"trajectories,":[58],"explicitly":[59],"marking":[60],"key":[62],"positions":[63],"that":[64],"require":[65],"correction.":[66],"Without":[67],"costly":[68],"large-scale":[69],"step-by-step":[70],"human":[71],"annotations,":[72],"our":[73],"method":[74],"enables":[75],"process-level":[76],"alignment":[78],"can":[80],"be":[81],"seamlessly":[82],"integrated":[83],"into":[84],"existing":[85],"GRPO-like":[86],"frameworks.":[87],"Experiments":[88],"multimodal":[90],"benchmarks":[92],"including":[93],"MMMStar":[94],"MathVista":[96],"show":[97],"an":[98,109],"average":[99],"3%":[100],"improvement":[101],"under":[102],"matched":[103],"compute":[104],"budgets.":[105],"Our":[106],"approach":[107],"offers":[108],"effective,":[110],"low-cost":[111],"solution":[112],"for":[113],"accurate":[114],"vision--reasoning":[115],"process":[116],"alignment.":[117]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-02T00:00:00"}