{"id":"https://openalex.org/W7225617212","doi":"https://doi.org/10.1145/3842498","title":"Steering Diffusion Models via Color-Activating Noise Patches for Controllable Image Synthesis","display_name":"Steering Diffusion Models via Color-Activating Noise Patches for Controllable Image Synthesis","publication_year":2026,"publication_date":"2026-11-09","ids":{"openalex":"https://openalex.org/W7225617212","doi":"https://doi.org/10.1145/3842498"},"language":"en","primary_location":{"id":"doi:10.1145/3842498","doi":"https://doi.org/10.1145/3842498","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3842498","pdf_url":null,"source":{"id":"https://openalex.org/S185367456","display_name":"ACM Transactions on Graphics","issn_l":"0730-0301","issn":["0730-0301","1557-7368"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","ccf-a","cwts-core","dongbi-a","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Graphics","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3842498","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138809719","display_name":"Rongjin Guo","orcid":"https://orcid.org/0009-0001-1602-0861","observed_orcids":["https://orcid.org/0009-0001-1602-0861"]},"institutions":[{"id":"https://openalex.org/I168719708","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23","country_code":"HK","type":"education","lineage":["https://openalex.org/I168719708"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Rongjin Guo","raw_affiliation_strings":["City University of Hong Kong, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0009-0001-1602-0861","affiliations":[{"raw_affiliation_string":"City University of Hong Kong, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I168719708"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025567094","display_name":"Ke Xu","orcid":"https://orcid.org/0000-0001-5855-3810","observed_orcids":["https://orcid.org/0000-0001-5855-3810","https://orcid.org/0000-0001-6266-4257","https://orcid.org/0009-0007-3532-619X"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ke Xu","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0001-5855-3810","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5006197413","display_name":"Rynson W. H. Lau","orcid":"https://orcid.org/0000-0002-8957-8129","observed_orcids":["https://orcid.org/0000-0002-8957-8129"]},"institutions":[{"id":"https://openalex.org/I4405263171","display_name":"City University of Hong Kong (Dongguan)","ror":"https://ror.org/03hj2gd47","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405263171"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rynson Lau","raw_affiliation_strings":["City University of Hong Kong (Dongguan), Dongguan, China"],"raw_orcid":"https://orcid.org/0000-0002-8957-8129","affiliations":[{"raw_affiliation_string":"City University of Hong Kong (Dongguan), Dongguan, China","institution_ids":["https://openalex.org/I4405263171"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"45","issue":"6","first_page":"1","last_page":"24"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/diffusion-models","display_name":"diffusion models","score":0.9070000052452087},{"id":"https://openalex.org/keywords/text-to-image-generation","display_name":"text-to-image generation","score":0.8420000076293945},{"id":"https://openalex.org/keywords/controllable-image-generation","display_name":"controllable image generation","score":0.7749999761581421},{"id":"https://openalex.org/keywords/diffusion-transformer","display_name":"diffusion transformer","score":0.7570000290870667},{"id":"https://openalex.org/keywords/interactive-image-editing","display_name":"interactive image editing","score":0.7369999885559082}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7663999795913696},{"id":"https://openalex.org/C48209547","wikidata":"https://www.wikidata.org/wiki/Q1331104","display_name":"Controllability","level":2,"score":0.6176999807357788},{"id":"https://openalex.org/C163294075","wikidata":"https://www.wikidata.org/wiki/Q581861","display_name":"Noise reduction","level":2,"score":0.5576000213623047},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.5471000075340271},{"id":"https://openalex.org/C2776674983","wikidata":"https://www.wikidata.org/wiki/Q545981","display_name":"Image editing","level":3,"score":0.5444999933242798},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.5440000295639038},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5435000061988831},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5257999897003174},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.453000009059906},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.4327999949455261},{"id":"https://openalex.org/C69357855","wikidata":"https://www.wikidata.org/wiki/Q163214","display_name":"Diffusion","level":2,"score":0.41110000014305115},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.39250001311302185},{"id":"https://openalex.org/C203504353","wikidata":"https://www.wikidata.org/wiki/Q4765461","display_name":"Anisotropic diffusion","level":3,"score":0.3806999921798706},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.36579999327659607},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.3646000027656555},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.36149999499320984},{"id":"https://openalex.org/C4199805","wikidata":"https://www.wikidata.org/wiki/Q2725903","display_name":"Gaussian noise","level":2,"score":0.32420000433921814},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3181999921798706},{"id":"https://openalex.org/C61224824","wikidata":"https://www.wikidata.org/wiki/Q2260434","display_name":"Mixture model","level":2,"score":0.314300000667572},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.302700012922287},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.29440000653266907},{"id":"https://openalex.org/C2776449333","wikidata":"https://www.wikidata.org/wiki/Q7928781","display_name":"View synthesis","level":3,"score":0.29109999537467957},{"id":"https://openalex.org/C9417928","wikidata":"https://www.wikidata.org/wiki/Q1070689","display_name":"Image processing","level":3,"score":0.27379998564720154},{"id":"https://openalex.org/C61326573","wikidata":"https://www.wikidata.org/wiki/Q1496376","display_name":"Gaussian process","level":3,"score":0.26759999990463257},{"id":"https://openalex.org/C125045340","wikidata":"https://www.wikidata.org/wiki/Q6002224","display_name":"Image formation","level":3,"score":0.257999986410141},{"id":"https://openalex.org/C35772409","wikidata":"https://www.wikidata.org/wiki/Q1323086","display_name":"Image noise","level":3,"score":0.2572999894618988},{"id":"https://openalex.org/C2983327147","wikidata":"https://www.wikidata.org/wiki/Q581861","display_name":"Image denoising","level":3,"score":0.25200000405311584},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.25110000371932983}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3842498","doi":"https://doi.org/10.1145/3842498","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3842498","pdf_url":null,"source":{"id":"https://openalex.org/S185367456","display_name":"ACM Transactions on Graphics","issn_l":"0730-0301","issn":["0730-0301","1557-7368"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","ccf-a","cwts-core","dongbi-a","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Graphics","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3842498","doi":"https://doi.org/10.1145/3842498","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3842498","pdf_url":null,"source":{"id":"https://openalex.org/S185367456","display_name":"ACM Transactions on Graphics","issn_l":"0730-0301","issn":["0730-0301","1557-7368"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","ccf-a","cwts-core","dongbi-a","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Graphics","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W1959608418","https://openalex.org/W3092442149","https://openalex.org/W3212616005","https://openalex.org/W4312933868","https://openalex.org/W4321473567","https://openalex.org/W4385848774","https://openalex.org/W4386076425","https://openalex.org/W4390872297","https://openalex.org/W4390873054","https://openalex.org/W4392909507","https://openalex.org/W4393968975","https://openalex.org/W4396650989","https://openalex.org/W4396945138","https://openalex.org/W4400582020","https://openalex.org/W4402716004","https://openalex.org/W4403998619","https://openalex.org/W4404450875","https://openalex.org/W4405094703","https://openalex.org/W4407794264","https://openalex.org/W4409262782","https://openalex.org/W4413147091","https://openalex.org/W4413147448","https://openalex.org/W4414580661","https://openalex.org/W6966874861","https://openalex.org/W7084573423","https://openalex.org/W7133231872","https://openalex.org/W7160172743"],"related_works":["https://openalex.org/W4416033243","https://openalex.org/W4407128381","https://openalex.org/W4404570196","https://openalex.org/W7139030105","https://openalex.org/W4399414064","https://openalex.org/W4402943798","https://openalex.org/W4321276804","https://openalex.org/W7169862152","https://openalex.org/W4413157747","https://openalex.org/W7138089739"],"abstract_inverted_index":{"While":[0],"diffusion":[1,130,157],"models":[2,158],"have":[3],"achieved":[4],"remarkable":[5],"progress":[6],"in":[7,28,49,91,102,167],"text-to-image":[8],"(T2I)":[9],"synthesis,":[10],"a":[11,46,118,127,136,228],"critical":[12],"question":[13,41],"remains:":[14],"how":[15],"can":[16,83,187,225],"we":[17,35,56,115,204],"achieve":[18],"precise":[19],"and":[20,26,44,74,132,141,174,200,211,238],"reliable":[21,168],"control":[22],"over":[23],"object":[24,198],"colors":[25],"positions":[27],"the":[29,50,69,88,92,96,105,108,153,162,193,217,236],"generated":[30],"images?":[31],"In":[32],"this":[33,40,113],"work,":[34],"first":[36],"aim":[37],"to":[38,121,189,215,231],"answer":[39],"by":[42,87],"uncovering":[43],"leveraging":[45],"novel":[47],"structure":[48],"initial":[51,70],"random":[52],"Gaussian":[53],"noise,":[54],"which":[55],"term":[57],"Color-Activating":[58],"Noise":[59],"(CAN)":[60],"patches.":[61,222],"These":[62],"CAN":[63,123,149,185,221],"patches":[64,124,150,186],"are":[65],"localized":[66],"regions":[67],"of":[68,107,155,161,219,240],"noise":[71],"that":[72,148,160,184],"consistently":[73],"intrinsically":[75],"couple":[76],"with":[77,159],"specific":[78],"colors,":[79],"i.e.":[80],",":[81],"they":[82],"be":[84],"reliably":[85],"activated":[86],"color-related":[89],"tokens":[90],"input":[93],"prompt":[94],"during":[95],"denoising":[97],"(generation)":[98],"process.":[99],"This":[100],"activation,":[101],"turn,":[103],"steers":[104],"position":[106,199],"associated":[109],"object.":[110],"Based":[111],"on":[112,234],"observation,":[114],"then":[116],"propose":[117],"mining":[119],"pipeline":[120],"discover":[122],"efficiently":[125],"from":[126],"pre-trained":[128],"UNet-based":[129,156],"model":[131],"organize":[133],"them":[134],"into":[135],"plug-and-play":[137],"database":[138],"for":[139,172],"retrieval":[140],"injection":[142],"at":[143],"inference":[144],"time.":[145],"We":[146,181],"demonstrate":[147],"successfully":[151],"align":[152],"capability":[154],"heavily-trained":[163],"Diffusion":[164],"Transformers":[165],"(DiTs)":[166],"color-position":[169],"control,":[170],"particularly":[171],"atypical":[173],"multi-color":[175],"generation":[176,194],"cases,":[177],"without":[178],"additional":[179],"optimization.":[180],"further":[182],"show":[183],"generalize":[188],"DiT-based":[190],"models,":[191],"improving":[192,235],"accuracy":[195],"via":[196],"better":[197],"color":[201],"control.":[202],"Finally,":[203],"showcase":[205],"two":[206],"applications,":[207],"interactive":[208],"image":[209,213],"editing":[210],"counting-aware":[212],"generation,":[214],"highlight":[216],"versatility":[218],"our":[220],"Our":[223],"method":[224],"serve":[226],"as":[227],"new":[229],"baseline":[230],"shed":[232],"light":[233],"controllability":[237],"faithfulness":[239],"diffusion-based":[241],"T2I":[242],"generation.":[243],"Project":[244],"page:":[245],"https://github.com/RongjinGuo/CAN-Patches.":[246]},"counts_by_year":[],"updated_date":"2026-11-11T12:16:26.678958","created_date":"2026-11-10T00:00:00"}