{"id":"https://openalex.org/W7168145625","doi":"https://doi.org/10.48550/arxiv.2607.09492","title":"Multimodal Reward Hacking in Reinforcement Learning","display_name":"Multimodal Reward Hacking in Reinforcement Learning","publication_year":2026,"publication_date":"2026-07-10","ids":{"openalex":"https://openalex.org/W7168145625","doi":"https://doi.org/10.48550/arxiv.2607.09492"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.09492","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.09492","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.09492","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140559280","display_name":"Jiayu Yao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Jiayu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140522971","display_name":"Yiwei Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yiwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090783536","display_name":"Anmeng Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Anmeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140521247","display_name":"Zhe Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Zhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064124461","display_name":"Songsong Wang","orcid":"https://orcid.org/0009-0000-7175-0747"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Songsong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140554977","display_name":"Lingrui Mei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mei, Lingrui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140550704","display_name":"Yuyao Ge","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ge, Yuyao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5103870133","display_name":"Liu S","orcid":"https://orcid.org/0000-0001-7255-0982"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Shenghua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5705000162124634,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5705000162124634,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.13179999589920044,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.061500001698732376,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7628999948501587},{"id":"https://openalex.org/keywords/hacker","display_name":"Hacker","score":0.578000009059906},{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.46650001406669617},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.44040000438690186},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.36469998955726624},{"id":"https://openalex.org/keywords/reinforcement","display_name":"Reinforcement","score":0.35760000348091125}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7628999948501587},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7014999985694885},{"id":"https://openalex.org/C86844869","wikidata":"https://www.wikidata.org/wiki/Q2798820","display_name":"Hacker","level":2,"score":0.578000009059906},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5342000126838684},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.46650001406669617},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.44040000438690186},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43549999594688416},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.36469998955726624},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.35760000348091125},{"id":"https://openalex.org/C2780310539","wikidata":"https://www.wikidata.org/wiki/Q12547192","display_name":"Imperfect","level":2,"score":0.29649999737739563},{"id":"https://openalex.org/C106934330","wikidata":"https://www.wikidata.org/wiki/Q1971873","display_name":"Trait","level":2,"score":0.2736000120639801},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.25920000672340393},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.25609999895095825},{"id":"https://openalex.org/C163164238","wikidata":"https://www.wikidata.org/wiki/Q2737027","display_name":"Failure rate","level":2,"score":0.25529998540878296}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.09492","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.09492","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.09492","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.09492","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"(RL)":[2],"is":[3,25,30,143,149,185],"increasingly":[4],"used":[5],"to":[6,162],"align":[7],"multimodal":[8,182],"large":[9],"language":[10],"models":[11],"(MLLMs),":[12],"but":[13,115],"higher":[14],"rewards":[15,88,134,165,197],"do":[16],"not":[17,117],"always":[18],"imply":[19],"better":[20],"task":[21],"performance.":[22],"This":[23],"risk":[24],"amplified":[26],"when":[27],"visual":[28],"evidence":[29],"evaluated":[31],"by":[32],"text-only":[33],"or":[34],"weakly":[35],"grounded":[36],"rewards.":[37],"We":[38,67],"study":[39],"reward":[40,54,81,183],"hacking":[41,184],"in":[42],"MLLM":[43],"RL":[44,62,104],"across":[45],"safety":[46],"VQA,":[47,49],"chart":[48],"and":[50,61,146,156,193,198],"stress-test":[51],"settings,":[52],"varying":[53],"design,":[55],"data":[56],"ambiguity,":[57],"model":[58,123],"scale":[59],"(2B-32B),":[60],"algorithm":[63],"(GRPO,":[64],"RLOO,":[65],"DAPO).":[66],"introduce":[68],"Newly":[69],"Rewarded":[70],"Failure":[71],"Rate":[72,96],"(NRFR),":[73],"which":[74],"measures":[75],"failures":[76,107],"among":[77],"samples":[78],"whose":[79],"proxy":[80],"improves":[82,158],"over":[83],"the":[84,121,136],"SFT":[85],"baseline.":[86],"Outcome-only":[87],"cause":[89],"severe":[90],"hacking,":[91,174],"reaching":[92],"48.1%":[93],"Reward":[94],"Hacking":[95],"(RHR),":[97],"while":[98,175],"NRFR":[99],"exceeding":[100],"RHR":[101],"shows":[102],"that":[103,200],"creates":[105],"new":[106],"rather":[108],"than":[109],"merely":[110],"inheriting":[111],"them.":[112],"Scaling":[113],"reduces":[114,179],"does":[116],"eliminate":[118],"hacking:":[119],"even":[120],"32B":[122],"retains":[124],"a":[125,186],"54.9%":[126],"worse":[127],"rate":[128],"under":[129,203],"outcome-only":[130],"rewards,":[131,192],"whereas":[132],"answer-aware":[133],"improve":[135],"oracle":[137],"trend":[138],"at":[139],"every":[140],"scale.":[141],"Robustness":[142],"also":[144],"algorithm-":[145],"scale-dependent:":[147],"GRPO":[148],"consistently":[150],"most":[151],"resistant,":[152],"RLOO":[153],"remains":[154],"vulnerable,":[155],"DAPO":[157],"substantially":[159],"from":[160],"2B":[161],"8B.":[163],"Visual-evidence":[164],"help":[166],"only":[167],"with":[168],"reliable":[169,202],"verification:":[170],"keyword-based":[171],"checks":[172],"increase":[173],"VLM-as-judge":[176],"semantic":[177],"verification":[178],"it.":[180],"Overall,":[181],"systematic":[187],"result":[188],"of":[189],"optimizing":[190],"imperfect":[191],"robust":[194],"alignment":[195],"requires":[196],"verifiers":[199],"remain":[201],"optimization":[204],"pressure.":[205]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-14T00:00:00"}