{"id":"https://openalex.org/W4385526465","doi":"https://doi.org/10.1145/3588432.3591549","title":"Modulating Pretrained Diffusion Models for Multimodal Image Synthesis","display_name":"Modulating Pretrained Diffusion Models for Multimodal Image Synthesis","publication_year":2023,"publication_date":"2023-07-19","ids":{"openalex":"https://openalex.org/W4385526465","doi":"https://doi.org/10.1145/3588432.3591549"},"language":"en","primary_location":{"id":"doi:10.1145/3588432.3591549","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3588432.3591549","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3588432.3591549","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Proceedings","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3588432.3591549","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029588803","display_name":"Cusuh Ham","orcid":"https://orcid.org/0009-0002-2686-052X"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cusuh Ham","raw_affiliation_strings":["Georgia Institute of Technology, United States of America"],"raw_orcid":"https://orcid.org/0009-0002-2686-052X","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, United States of America","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016775802","display_name":"James Hays","orcid":"https://orcid.org/0000-0001-7016-4252"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James Hays","raw_affiliation_strings":["Georgia Institute of Technology, United States of America"],"raw_orcid":"https://orcid.org/0000-0001-7016-4252","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, United States of America","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085050428","display_name":"Jingwan Lu","orcid":"https://orcid.org/0000-0002-3598-9918"},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jingwan Lu","raw_affiliation_strings":["Adobe Research, United States of America"],"raw_orcid":"https://orcid.org/0000-0002-3598-9918","affiliations":[{"raw_affiliation_string":"Adobe Research, United States of America","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101960524","display_name":"Krishna Kumar Singh","orcid":"https://orcid.org/0000-0002-8066-6835"},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Krishna Kumar Singh","raw_affiliation_strings":["Adobe Research, United States of America"],"raw_orcid":"https://orcid.org/0000-0002-8066-6835","affiliations":[{"raw_affiliation_string":"Adobe Research, United States of America","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114860359","display_name":"Zhifei Zhang","orcid":"https://orcid.org/0000-0003-0466-9548"},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhifei Zhang","raw_affiliation_strings":["Adobe Research, United States of America"],"raw_orcid":"https://orcid.org/0000-0003-0466-9548","affiliations":[{"raw_affiliation_string":"Adobe Research, United States of America","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001873096","display_name":"Tobias Hinz","orcid":"https://orcid.org/0000-0003-1354-1562"},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tobias Hinz","raw_affiliation_strings":["Adobe Research, United States of America"],"raw_orcid":"https://orcid.org/0000-0003-1354-1562","affiliations":[{"raw_affiliation_string":"Adobe Research, United States of America","institution_ids":["https://openalex.org/I1306409833"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9836999773979187,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8114354610443115},{"id":"https://openalex.org/keywords/diffusion","display_name":"Diffusion","score":0.6580178141593933},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.596886157989502},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.5278603434562683},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4973626434803009},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.49480074644088745},{"id":"https://openalex.org/keywords/diffusion-process","display_name":"Diffusion process","score":0.44731295108795166},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.37388184666633606},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.06258326768875122}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8114354610443115},{"id":"https://openalex.org/C69357855","wikidata":"https://www.wikidata.org/wiki/Q163214","display_name":"Diffusion","level":2,"score":0.6580178141593933},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.596886157989502},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.5278603434562683},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4973626434803009},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.49480074644088745},{"id":"https://openalex.org/C68710425","wikidata":"https://www.wikidata.org/wiki/Q5275442","display_name":"Diffusion process","level":3,"score":0.44731295108795166},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.37388184666633606},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.06258326768875122},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.0},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C3017618536","wikidata":"https://www.wikidata.org/wiki/Q304994","display_name":"Innovation diffusion","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3588432.3591549","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3588432.3591549","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3588432.3591549","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Proceedings","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3588432.3591549","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3588432.3591549","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3588432.3591549","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Proceedings","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385526465.pdf","grobid_xml":"https://content.openalex.org/works/W4385526465.grobid-xml"},"referenced_works_count":27,"referenced_works":["https://openalex.org/W569478347","https://openalex.org/W845365781","https://openalex.org/W1834627138","https://openalex.org/W1861492603","https://openalex.org/W2412782625","https://openalex.org/W2473464331","https://openalex.org/W2886934227","https://openalex.org/W2962770929","https://openalex.org/W2962785568","https://openalex.org/W2962967409","https://openalex.org/W2962974533","https://openalex.org/W2963073614","https://openalex.org/W3035574324","https://openalex.org/W3036167779","https://openalex.org/W3162926177","https://openalex.org/W3174194560","https://openalex.org/W3176913662","https://openalex.org/W3180355996","https://openalex.org/W3212516020","https://openalex.org/W4281551249","https://openalex.org/W4287210973","https://openalex.org/W4288083516","https://openalex.org/W4296193306","https://openalex.org/W4312529868","https://openalex.org/W4312933868","https://openalex.org/W6779823529","https://openalex.org/W6795288823"],"related_works":["https://openalex.org/W2386796262","https://openalex.org/W2354853573","https://openalex.org/W2064306826","https://openalex.org/W2807797544","https://openalex.org/W2924020013","https://openalex.org/W2057463076","https://openalex.org/W2383408482","https://openalex.org/W1511383268","https://openalex.org/W2614023814","https://openalex.org/W2097757605"],"abstract_inverted_index":{"We":[0,89,154],"present":[1],"multimodal":[2,16],"conditioning":[3,179],"modules":[4],"(MCM)":[5],"for":[6,35],"enabling":[7],"conditional":[8],"image":[9,102,110],"synthesis":[10,17],"using":[11,70,146],"pretrained":[12,27,43],"diffusion":[13,38,53,65,87,126,141],"models.":[14,39],"Previous":[15],"works":[18],"rely":[19],"on":[20,158],"training":[21,84,152],"networks":[22,44],"from":[23,123],"scratch":[24],"or":[25],"fine-tuning":[26],"networks,":[28],"both":[29],"of":[30,85,100,129,133,136,138,151],"which":[31],"are":[32],"computationally":[33],"expensive":[34],"large,":[36],"state-of-the-art":[37],"Our":[40],"method":[41,157],"uses":[42],"but":[45],"does":[46,119],"not":[47,120],"require":[48,121],"any":[49],"updates":[50],"to":[51,62,105,163,177],"the":[52,64,82,86,97,101,109,124,134,139,165,169,178],"network\u2019s":[54,66],"parameters.":[55],"MCM":[56,92,114],"is":[57,115,144],"a":[58,148],"small":[59],"module":[60],"trained":[61,145],"modulate":[63],"predictions":[67],"during":[68,81],"sampling":[69],"2D":[71],"modalities":[72],"(e.g.,":[73],"semantic":[74],"segmentation":[75],"maps,":[76],"sketches)":[77],"that":[78,91],"were":[79],"unseen":[80],"original":[83,125],"model.":[88],"show":[90],"enables":[93],"user":[94],"control":[95,107,167],"over":[96,108,168],"spatial":[98],"layout":[99],"and":[103,143,160,172],"leads":[104],"increased":[106],"generation":[111],"process.":[112],"Training":[113],"cheap":[116],"as":[117],"it":[118],"gradients":[122],"net,":[127],"consists":[128],"only":[130,147],"\u223c":[131],"1%":[132],"number":[135,150],"parameters":[137],"base":[140],"model,":[142],"limited":[149],"examples.":[153],"evaluate":[155],"our":[156],"unconditional":[159],"text-conditional":[161],"models":[162],"demonstrate":[164],"improved":[166],"generated":[170],"images":[171],"their":[173],"alignment":[174],"with":[175],"respect":[176],"inputs.":[180]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}