{"id":"https://openalex.org/W7161705746","doi":"https://doi.org/10.48550/arxiv.2605.16842","title":"Sketch Then Paint: Hierarchical Reinforcement Learning for Diffusion Multi-Modal Large Language Models","display_name":"Sketch Then Paint: Hierarchical Reinforcement Learning for Diffusion Multi-Modal Large Language Models","publication_year":2026,"publication_date":"2026-05-16","ids":{"openalex":"https://openalex.org/W7161705746","doi":"https://doi.org/10.48550/arxiv.2605.16842"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.16842","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.16842","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.16842","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136491868","display_name":"Siqi Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Siqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090122335","display_name":"Jianghan Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Jianghan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136498001","display_name":"Yi Xin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xin, Yi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128813361","display_name":"Huayu Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Huayu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136493309","display_name":"Haoxing Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Haoxing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029140686","display_name":"Yan Tai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tai, Yan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136489437","display_name":"Yue Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136487258","display_name":"Junjun He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Junjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136470178","display_name":"Yihao Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yihao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136478891","display_name":"Guangtao Zhai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhai, Guangtao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002643299","display_name":"Y. Cao","orcid":"https://orcid.org/0000-0003-0858-4488"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Yuewen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136490601","display_name":"Xiaohong Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xiaohong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5286999940872192,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5286999940872192,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.20160000026226044,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0731000006198883,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7391999959945679},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.6632000207901001},{"id":"https://openalex.org/keywords/hierarchy","display_name":"Hierarchy","score":0.5956000089645386},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.557200014591217},{"id":"https://openalex.org/keywords/sketch","display_name":"Sketch","score":0.5062000155448914},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.49549999833106995},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.48350000381469727},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4740000069141388}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8043000102043152},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7391999959945679},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6632000207901001},{"id":"https://openalex.org/C31170391","wikidata":"https://www.wikidata.org/wiki/Q188619","display_name":"Hierarchy","level":2,"score":0.5956000089645386},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5891000032424927},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.557200014591217},{"id":"https://openalex.org/C2779231336","wikidata":"https://www.wikidata.org/wiki/Q7534724","display_name":"Sketch","level":2,"score":0.5062000155448914},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.49549999833106995},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.48350000381469727},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4740000069141388},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46389999985694885},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.4458000063896179},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.39500001072883606},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.38659998774528503},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.38370001316070557},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.3361999988555908},{"id":"https://openalex.org/C144986985","wikidata":"https://www.wikidata.org/wiki/Q871236","display_name":"Hierarchical database model","level":2,"score":0.3264000117778778},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.31380000710487366},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.3084000051021576},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.2840999960899353},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.2581999897956848}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.16842","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.16842","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.16842","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.16842","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Diffusion":[0],"Multi-Modal":[1],"Large":[2],"Language":[3],"Models":[4],"(dMLLMs)":[5],"are":[6],"powerful":[7],"for":[8],"image":[9,29,196],"generation,":[10],"but":[11],"optimizing":[12],"them":[13],"through":[14,33],"reinforcement":[15],"learning":[16],"(RL)":[17],"remains":[18],"a":[19,27,117,135,144,151],"major":[20],"challenge.":[21],"One":[22],"primary":[23],"difficulty":[24],"is":[25],"that":[26,121,156,176],"single":[28],"can":[30],"be":[31],"generated":[32],"many":[34],"different":[35],"unmasking":[36],"sequences,":[37],"which":[38,104],"makes":[39],"calculating":[40],"importance":[41,140],"ratios":[42,141],"often":[43],"intractable.":[44],"Additionally,":[45],"existing":[46],"methods":[47,80],"tend":[48],"to":[49,75,82,90,138,161],"ignore":[50],"the":[51,61,84,91,110,182],"hierarchical":[52],"generation":[53],"process":[54],"of":[55,87],"dMLLMs,":[56],"where":[57],"early":[58],"tokens":[59,66,160],"define":[60],"global":[62],"layout":[63],"and":[64,130,173,184,199],"later":[65],"focus":[67],"on":[68,181],"local":[69],"details.":[70],"By":[71],"assigning":[72],"uniform":[73],"rewards":[74],"all":[76],"tokens,":[77],"these":[78,96],"current":[79],"fail":[81],"reflect":[83],"actual":[85],"contribution":[86],"each":[88],"token":[89],"final":[92],"image.":[93],"To":[94],"address":[95],"issues,":[97],"we":[98,149],"propose":[99],"Hierarchical":[100,152],"Token":[101],"GRPO":[102],"(HT-GRPO),":[103],"integrates":[105],"this":[106],"hierarchy":[107],"directly":[108],"into":[109,124],"policy":[111],"optimization":[112],"process.":[113],"Our":[114],"approach":[115],"features":[116],"Sketch-Then-Paint":[118],"training":[119],"scheme":[120],"organizes":[122],"updates":[123],"three":[125],"distinct":[126],"stages:":[127],"global,":[128],"structure,":[129],"refinement.":[131],"We":[132],"also":[133],"use":[134],"prompt-conditioned":[136],"estimator":[137],"calculate":[139],"starting":[142],"from":[143],"fully":[145],"masked":[146],"state.":[147],"Furthermore,":[148],"introduce":[150],"Credit":[153],"Assignment":[154],"mechanism":[155],"prioritizes":[157],"key":[158],"structural":[159],"ensure":[162],"accurate":[163],"reward":[164],"propagation.":[165],"Experiments":[166],"using":[167],"two":[168],"popular":[169],"dMLLM":[170],"backbones,":[171],"MMaDA":[172],"Lumina-DiMOO,":[174],"demonstrate":[175],"HT-GRPO":[177],"achieves":[178],"substantial":[179],"gains":[180],"GenEval":[183],"DPG":[185],"benchmarks.":[186],"Evaluations":[187],"across":[188],"six":[189],"additional":[190],"metrics":[191],"confirm":[192],"significant":[193],"improvements":[194],"in":[195],"quality,":[197],"aesthetics,":[198],"human":[200],"preference.":[201]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-20T00:00:00"}