{"id":"https://openalex.org/W4401249388","doi":"https://doi.org/10.1145/3686155","title":"Multi-Model Style-Aware Diffusion Learning for Semantic Image Synthesis","display_name":"Multi-Model Style-Aware Diffusion Learning for Semantic Image Synthesis","publication_year":2024,"publication_date":"2024-08-02","ids":{"openalex":"https://openalex.org/W4401249388","doi":"https://doi.org/10.1145/3686155"},"language":"en","primary_location":{"id":"doi:10.1145/3686155","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3686155","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3686155","source":{"id":"https://openalex.org/S19610489","display_name":"ACM Transactions on Multimedia Computing Communications and Applications","issn_l":"1551-6857","issn":["1551-6857","1551-6865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Multimedia Computing, Communications, and Applications","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3686155","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101319174","display_name":"Yunfang Niu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yunfang Niu","raw_affiliation_strings":["Foundation Model Research Center, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Foundation Model Research Center, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061646454","display_name":"Lingxiang Wu","orcid":"https://orcid.org/0000-0001-9346-3597"},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lingxiang Wu","raw_affiliation_strings":["Foundation Model Research Center, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Foundation Model Research Center, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091947285","display_name":"Yufeng Zhang","orcid":"https://orcid.org/0000-0003-4288-5022"},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yufeng Zhang","raw_affiliation_strings":["Foundation Model Research Center, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Foundation Model Research Center, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065043476","display_name":"Yousong Zhu","orcid":"https://orcid.org/0000-0001-8544-410X"},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yousong Zhu","raw_affiliation_strings":["Foundation Model Research Center, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Foundation Model Research Center, Institute of Automation, Chinese Academy of Sciences, Beijing, China and School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029271867","display_name":"Guibo Zhu","orcid":"https://orcid.org/0000-0001-8293-3952"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guibo Zhu","raw_affiliation_strings":["Foundation Model Research Center, Institute of Automation, Chinese Academy of Sciences, Beijing, China, School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China, and Shanghai Artificial Intelligence Laboratory, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Foundation Model Research Center, Institute of Automation, Chinese Academy of Sciences, Beijing, China, School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China, and Shanghai Artificial Intelligence Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4210112150","https://openalex.org/I19820366"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5058420913","display_name":"Jinqiao Wang","orcid":"https://orcid.org/0000-0002-9118-2780"},"institutions":[{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jinqiao Wang","raw_affiliation_strings":["Foundation Model Research Center, Institute of Automation, Chinese Academy of Sciences, Beijing, China, School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China, and Peng Cheng Laboratory, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Foundation Model Research Center, Institute of Automation, Chinese Academy of Sciences, Beijing, China, School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China, and Peng Cheng Laboratory, Shenzhen, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I4210165038","https://openalex.org/I4210136793"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5101319174"],"corresponding_institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I4210165038"],"apc_list":null,"apc_paid":null,"fwci":0.7479,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.71378247,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":"20","issue":"11","first_page":"1","last_page":"21"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10481","display_name":"Computer Graphics and Visualization Techniques","score":0.988099992275238,"subfield":{"id":"https://openalex.org/subfields/1704","display_name":"Computer Graphics and Computer-Aided Design"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9715999960899353,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8735334873199463},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.5651909112930298},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5444215536117554},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4878256618976593},{"id":"https://openalex.org/keywords/autoregressive-model","display_name":"Autoregressive model","score":0.4570254981517792},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.44413459300994873},{"id":"https://openalex.org/keywords/image-editing","display_name":"Image editing","score":0.42936980724334717},{"id":"https://openalex.org/keywords/image-synthesis","display_name":"Image synthesis","score":0.41623762249946594},{"id":"https://openalex.org/keywords/fidelity","display_name":"Fidelity","score":0.4107094407081604},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3480662703514099},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.33707374334335327},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.09381791949272156}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8735334873199463},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.5651909112930298},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5444215536117554},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4878256618976593},{"id":"https://openalex.org/C159877910","wikidata":"https://www.wikidata.org/wiki/Q2202883","display_name":"Autoregressive model","level":2,"score":0.4570254981517792},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.44413459300994873},{"id":"https://openalex.org/C2776674983","wikidata":"https://www.wikidata.org/wiki/Q545981","display_name":"Image editing","level":3,"score":0.42936980724334717},{"id":"https://openalex.org/C2989087649","wikidata":"https://www.wikidata.org/wiki/Q176953","display_name":"Image synthesis","level":3,"score":0.41623762249946594},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.4107094407081604},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3480662703514099},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.33707374334335327},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.09381791949272156},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3686155","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3686155","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3686155","source":{"id":"https://openalex.org/S19610489","display_name":"ACM Transactions on Multimedia Computing Communications and Applications","issn_l":"1551-6857","issn":["1551-6857","1551-6865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Multimedia Computing, Communications, and Applications","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3686155","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3686155","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3686155","source":{"id":"https://openalex.org/S19610489","display_name":"ACM Transactions on Multimedia Computing Communications and Applications","issn_l":"1551-6857","issn":["1551-6857","1551-6865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Multimedia Computing, Communications, and Applications","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/11","display_name":"Sustainable cities and communities","score":0.7900000214576721}],"awards":[{"id":"https://openalex.org/G3464411458","display_name":null,"funder_award_id":"2022ZD0160601","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G6506700831","display_name":null,"funder_award_id":"62306315","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4401249388.pdf"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W1901129140","https://openalex.org/W2340897893","https://openalex.org/W2737258237","https://openalex.org/W2891331468","https://openalex.org/W2909012790","https://openalex.org/W2951848755","https://openalex.org/W2962974533","https://openalex.org/W2963073614","https://openalex.org/W2963800363","https://openalex.org/W2990269423","https://openalex.org/W3037451952","https://openalex.org/W3106333289","https://openalex.org/W3134430547","https://openalex.org/W3167154867","https://openalex.org/W3215495615","https://openalex.org/W3217132494","https://openalex.org/W4214896365","https://openalex.org/W4221139906","https://openalex.org/W4226125322","https://openalex.org/W4283805732","https://openalex.org/W4286850199","https://openalex.org/W4288062174","https://openalex.org/W4289766616","https://openalex.org/W4297645505","https://openalex.org/W4306676681","https://openalex.org/W4312933868","https://openalex.org/W4313499710","https://openalex.org/W4319300158","https://openalex.org/W4361271773","https://openalex.org/W4387969545","https://openalex.org/W4390873054","https://openalex.org/W6600446476","https://openalex.org/W6755234996","https://openalex.org/W6804122398","https://openalex.org/W6839517220"],"related_works":["https://openalex.org/W4365211920","https://openalex.org/W3014948380","https://openalex.org/W4380551139","https://openalex.org/W4317695495","https://openalex.org/W4287117424","https://openalex.org/W4387506531","https://openalex.org/W4238433571","https://openalex.org/W3174044702","https://openalex.org/W2967848559","https://openalex.org/W4299831724"],"abstract_inverted_index":{"Semantic":[0],"image":[1,61,86,102],"synthesis":[2],"aims":[3],"to":[4,20,69,76,121],"generate":[5],"images":[6,81,161],"from":[7,126],"given":[8],"semantic":[9,101],"layouts,":[10],"which":[11],"is":[12,48,55,119,201],"a":[13,93,105,109,116,127,149,155],"challenging":[14],"task":[15],"that":[16,198],"requires":[17],"training":[18,46,106,114,139],"models":[19],"capture":[21],"the":[22,43,51,59,64,70,113,123,138,144],"relationship":[23],"between":[24],"layouts":[25],"and":[26,50,63,82,108,152,170,180,185,209],"images.":[27],"Previous":[28],"works":[29],"are":[30],"usually":[31],"based":[32],"on":[33,177],"Generative":[34],"Adversarial":[35],"Networks":[36],"(GAN)":[37],"or":[38],"autoregressive":[39],"(AR)":[40],"models.":[41],"However,":[42],"GAN":[44],"model's":[45],"process":[47,140],"unstable,":[49],"AR":[52],"model\u2019s":[53],"performance":[54],"seriously":[56],"affected":[57],"by":[58],"independent":[60],"encoder":[62],"unidirectional":[65],"generation":[66],"bias.":[67],"Due":[68],"above":[71],"limitations,":[72],"these":[73],"methods":[74],"tend":[75],"synthesize":[77],"unrealistic,":[78],"poorly":[79],"aligned":[80],"only":[83],"consider":[84],"single-style":[85],"generation.":[87],"In":[88,112,143],"this":[89],"paper,":[90],"we":[91,147],"propose":[92],"Multi-model":[94],"Style-aware":[95],"Diffusion":[96],"Learning":[97],"(MSDL)":[98],"framework":[99],"for":[100,159],"synthesis,":[103],"including":[104],"module":[107],"sampling":[110,145],"module.":[111],"module,":[115,146],"layout-to-image":[117],"model":[118,128,200],"introduced":[120],"transfer":[122],"learned":[124],"knowledge":[125],"pretrained":[129],"with":[130,187],"massive":[131],"weak":[132],"correlated":[133],"text-image":[134],"pairs":[135],"data,":[136],"making":[137,182],"more":[141],"efficient.":[142],"designed":[148,154],"map-guidance":[150],"technique":[151],"creatively":[153],"multi-model":[156],"style-guidance":[157],"strategy":[158],"creating":[160],"in":[162,205],"multiple":[163,188],"styles,":[164],"e.g.,":[165],"oil":[166],"painting,":[167],"Disney":[168],"Cartoon,":[169],"pixel":[171],"style.":[172],"We":[173],"evaluate":[174],"our":[175,199],"method":[176],"Cityscapes,":[178],"ADE20K,":[179],"COCO-Stuff,":[181],"visual":[183],"comparisons":[184],"computing":[186],"metrics":[189],"such":[190],"as":[191],"FID,":[192],"LPIPS,":[193],"etc.":[194],"Experimental":[195],"results":[196],"demonstrate":[197],"highly":[202],"competitive,":[203],"especially":[204],"terms":[206],"of":[207],"fidelity":[208],"diversity.":[210]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}