{"id":"https://openalex.org/W4384519430","doi":"https://doi.org/10.1109/tmm.2023.3295094","title":"Utilizing Greedy Nature for Multimodal Conditional Image Synthesis in Transformers","display_name":"Utilizing Greedy Nature for Multimodal Conditional Image Synthesis in Transformers","publication_year":2023,"publication_date":"2023-07-17","ids":{"openalex":"https://openalex.org/W4384519430","doi":"https://doi.org/10.1109/tmm.2023.3295094"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2023.3295094","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3295094","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051338650","display_name":"Sitong Su","orcid":"https://orcid.org/0009-0001-7406-116X"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sitong Su","raw_affiliation_strings":["Center for Future Media, University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":"https://orcid.org/0009-0001-7406-116X","affiliations":[{"raw_affiliation_string":"Center for Future Media, University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087225139","display_name":"Junchen Zhu","orcid":"https://orcid.org/0000-0002-3872-6689"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junchen Zhu","raw_affiliation_strings":["Center for Future Media, University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":"https://orcid.org/0000-0002-3872-6689","affiliations":[{"raw_affiliation_string":"Center for Future Media, University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066645546","display_name":"Lianli Gao","orcid":"https://orcid.org/0000-0002-2522-6394"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lianli Gao","raw_affiliation_strings":["Center for Future Media, University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":"https://orcid.org/0000-0002-2522-6394","affiliations":[{"raw_affiliation_string":"Center for Future Media, University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036987388","display_name":"Jingkuan Song","orcid":"https://orcid.org/0000-0002-2549-8322"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingkuan Song","raw_affiliation_strings":["Institute of Neurology, Sichuan Provincial People&#x0027;s Hospital, University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":"https://orcid.org/0000-0002-2549-8322","affiliations":[{"raw_affiliation_string":"Institute of Neurology, Sichuan Provincial People&#x0027;s Hospital, University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I150229711"],"apc_list":null,"apc_paid":null,"fwci":0.8509,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.75017834,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"26","issue":null,"first_page":"2354","last_page":"2366"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.878966212272644},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7455977201461792},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.704910933971405},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5507361888885498},{"id":"https://openalex.org/keywords/greedy-algorithm","display_name":"Greedy algorithm","score":0.44910335540771484},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"Multimodal learning","score":0.4147261679172516},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3946996033191681},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.37557271122932434},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3368913531303406},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.19522178173065186}],"concepts":[{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.878966212272644},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7455977201461792},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.704910933971405},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5507361888885498},{"id":"https://openalex.org/C51823790","wikidata":"https://www.wikidata.org/wiki/Q504353","display_name":"Greedy algorithm","level":2,"score":0.44910335540771484},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.4147261679172516},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3946996033191681},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37557271122932434},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3368913531303406},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.19522178173065186},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2023.3295094","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3295094","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2546663186","display_name":null,"funder_award_id":"62020106008","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6864684409","display_name":null,"funder_award_id":"62122018","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6971015501","display_name":null,"funder_award_id":"171106","funder_id":"https://openalex.org/F4320334945","funder_display_name":"Fok Ying Tong Education Foundation"},{"id":"https://openalex.org/G7314976618","display_name":"\u878d\u5408\u81ea\u7136\u8bed\u8a00\u5904\u7406\u7684\u6df1\u5ea6\u89c6\u89c9\u7406\u89e3\u5173\u952e\u6280\u672f\u7814\u7a76","funder_award_id":"61872064","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8027008232","display_name":"\u534f\u540c\u6df1\u5ea6\u89c6\u9891\u7406\u89e3\u3001\u63cf\u8ff0\u548c\u89c6\u89c9\u95ee\u7b54\u7684\u5173\u952e\u6280\u672f\u7814\u7a76","funder_award_id":"61772116","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320334945","display_name":"Fok Ying Tong Education Foundation","ror":"https://ror.org/01mv9t934"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":81,"referenced_works":["https://openalex.org/W753847829","https://openalex.org/W1821462560","https://openalex.org/W2133665775","https://openalex.org/W2507296351","https://openalex.org/W2614580201","https://openalex.org/W2886300652","https://openalex.org/W2895340641","https://openalex.org/W2962770929","https://openalex.org/W2962974533","https://openalex.org/W2963184176","https://openalex.org/W2964295764","https://openalex.org/W2981537897","https://openalex.org/W2987861506","https://openalex.org/W3016121695","https://openalex.org/W3034203264","https://openalex.org/W3034269985","https://openalex.org/W3174194560","https://openalex.org/W3176157254","https://openalex.org/W3180355996","https://openalex.org/W3190537575","https://openalex.org/W3195154899","https://openalex.org/W3205398323","https://openalex.org/W3206384369","https://openalex.org/W3207212285","https://openalex.org/W3209229003","https://openalex.org/W3215495615","https://openalex.org/W4224035735","https://openalex.org/W4226017838","https://openalex.org/W4226055713","https://openalex.org/W4226177592","https://openalex.org/W4246193833","https://openalex.org/W4283705858","https://openalex.org/W4283805732","https://openalex.org/W4284703076","https://openalex.org/W4285602612","https://openalex.org/W4295036294","https://openalex.org/W4296193306","https://openalex.org/W4304092645","https://openalex.org/W4312388283","https://openalex.org/W4312402957","https://openalex.org/W4312529868","https://openalex.org/W4312626422","https://openalex.org/W4312639100","https://openalex.org/W4312911498","https://openalex.org/W4312933868","https://openalex.org/W4313021454","https://openalex.org/W4313036289","https://openalex.org/W4313855701","https://openalex.org/W4319663704","https://openalex.org/W4320008790","https://openalex.org/W4320018929","https://openalex.org/W4321380967","https://openalex.org/W4382457527","https://openalex.org/W4386071687","https://openalex.org/W6638523607","https://openalex.org/W6762403277","https://openalex.org/W6764756247","https://openalex.org/W6765779288","https://openalex.org/W6767457696","https://openalex.org/W6768853884","https://openalex.org/W6771918413","https://openalex.org/W6779823529","https://openalex.org/W6784187948","https://openalex.org/W6784500203","https://openalex.org/W6790978476","https://openalex.org/W6791353385","https://openalex.org/W6795785581","https://openalex.org/W6796242362","https://openalex.org/W6797139422","https://openalex.org/W6797528369","https://openalex.org/W6802988923","https://openalex.org/W6809885388","https://openalex.org/W6810226654","https://openalex.org/W6810447287","https://openalex.org/W6810814029","https://openalex.org/W6810940779","https://openalex.org/W6838639034","https://openalex.org/W6839091850","https://openalex.org/W6839517220","https://openalex.org/W6843591305","https://openalex.org/W6849847750"],"related_works":["https://openalex.org/W2962931510","https://openalex.org/W4380551887","https://openalex.org/W2904518532","https://openalex.org/W3200817606","https://openalex.org/W4381827277","https://openalex.org/W3157841754","https://openalex.org/W4291167332","https://openalex.org/W2963685106","https://openalex.org/W3118761370","https://openalex.org/W2996247699"],"abstract_inverted_index":{"Multimodal":[0],"Conditional":[1],"Image":[2],"Synthesis(MCIS)":[3],"aims":[4],"to":[5,9,19,60,118,134,140,171],"generate":[6],"images":[7],"according":[8],"different":[10,211],"modalities":[11,45,55,129,148,153],"input":[12],"and":[13,30,49,58,100,130],"their":[14,21,62,132],"combination,":[15],"which":[16,123],"allows":[17],"users":[18],"describe":[20],"requirements":[22],"in":[23,37,106,114],"complementary":[24,63,208],"ways,":[25],"e.g.":[26],"segmentation":[27],"for":[28,32,151],"shapes":[29],"text":[31],"attributes.":[33],"Despite":[34],"satisfying":[35],"results":[36],"MCIS,":[38],"a":[39,103,196],"non-trivial":[40],"issue":[41],"is":[42,117],"neglected.":[43],"Some":[44],"are":[46,56,165],"fully":[47,206],"optimized":[48],"dominate":[50],"the":[51,85,93,120,125,142,174],"generation,":[52],"while":[53],"other":[54],"sub-optimized":[57,152],"fail":[59],"contribute":[61],"information.":[64],"We":[65],"coin":[66],"this":[67],"phenomenon":[68],"as":[69,149],"Modality":[70,115],"Bias.":[71],"<italic":[72],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[73],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Our":[74],"analysis":[75],"reveals":[76],"that":[77,87,187],"generative":[78],"models":[79],"own":[80],"greedy":[81,121,143],"nature</i>":[82],".":[83],"Specifically,":[84],"modality":[86,95],"shares":[88],"less":[89],"semantic":[90],"gap":[91],"with":[92],"synthesized":[94],"will":[96],"be":[97],"greedily":[98],"incorporated":[99],"thus":[101],"takes":[102],"larger":[104],"proportion":[105],"synthesis.":[107,136],"The":[108],"main":[109],"idea":[110],"of":[111,127,176],"previous":[112],"works":[113],"Bias":[116],"punish":[119],"nature,":[122],"hurts":[124],"performance":[126,175,194],"dominant":[128,147],"impedes":[131],"contribution":[133],"multimodal":[135,177,202],"Instead,":[137],"we":[138],"propose":[139],"utilize":[141],"nature":[144],"by":[145,167,195,205],"setting":[146],"guidance":[150],"through":[154],"coordinated":[155,169],"feature":[156],"space,":[157],"named":[158],"Coordinated":[159,181],"Knowledge":[160,182],"Mining.":[161],"Afterwards,":[162],"improved":[163],"uni-modalities":[164],"aggregated":[166],"fusing":[168],"features":[170],"further":[172],"boost":[173],"image":[178,203],"synthesis,":[179],"called":[180],"Fusion.":[183],"Extensive":[184],"experiments":[185],"prove":[186],"our":[188],"method":[189],"not":[190],"only":[191],"increases":[192],"uni-modal":[193],"large":[197],"margin,":[198],"but":[199],"also":[200],"promotes":[201],"synthesis":[204],"utilizing":[207],"information":[209],"from":[210],"modalities.":[212]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":3}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}