{"id":"https://openalex.org/W4403791527","doi":"https://doi.org/10.1145/3664647.3686836","title":"Dynamic Prompting of Frozen Text-to-Image Diffusion Models for Panoptic Narrative Grounding","display_name":"Dynamic Prompting of Frozen Text-to-Image Diffusion Models for Panoptic Narrative Grounding","publication_year":2024,"publication_date":"2024-10-26","ids":{"openalex":"https://openalex.org/W4403791527","doi":"https://doi.org/10.1145/3664647.3686836"},"language":"en","primary_location":{"id":"doi:10.1145/3664647.3686836","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3686836","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5106554007","display_name":"Hongyu Li","orcid":"https://orcid.org/0009-0001-9261-4012"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Hongyu Li","raw_affiliation_strings":["School of Artificial Intelligence, Beihang University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056811650","display_name":"Tianrui Hui","orcid":"https://orcid.org/0000-0002-1172-1554"},"institutions":[{"id":"https://openalex.org/I16365422","display_name":"Hefei University of Technology","ror":"https://ror.org/02czkny70","country_code":"CN","type":"education","lineage":["https://openalex.org/I16365422"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianrui Hui","raw_affiliation_strings":["School of Computer Science and Information Engineering, Hefei University of Technology, Hefei, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Information Engineering, Hefei University of Technology, Hefei, China","institution_ids":["https://openalex.org/I16365422"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000101368","display_name":"Zihan Ding","orcid":"https://orcid.org/0000-0002-9863-5754"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zihan Ding","raw_affiliation_strings":["School of Artificial Intelligence, Beihang University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052709846","display_name":"Jing Zhang","orcid":"https://orcid.org/0000-0003-3516-0111"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jing Zhang","raw_affiliation_strings":["School of Software, Beihang University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"School of Software, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080757156","display_name":"Bin Ma","orcid":"https://orcid.org/0000-0001-8371-0590"},"institutions":[{"id":"https://openalex.org/I4210087373","display_name":"Meizu (China)","ror":"https://ror.org/0067g4302","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210087373"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Ma","raw_affiliation_strings":["Meituan, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Meituan, Beijing, China","institution_ids":["https://openalex.org/I4210087373"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069204507","display_name":"Xiaoming Wei","orcid":null},"institutions":[{"id":"https://openalex.org/I4210087373","display_name":"Meizu (China)","ror":"https://ror.org/0067g4302","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210087373"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoming Wei","raw_affiliation_strings":["Meituan, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Meituan, Beijing, China","institution_ids":["https://openalex.org/I4210087373"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082300353","display_name":"Jizhong Han","orcid":"https://orcid.org/0000-0003-1107-3873"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"funder","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jizhong Han","raw_affiliation_strings":["Institute of Information Engineering, Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Institute of Information Engineering, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I19820366"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100330138","display_name":"Si Liu","orcid":"https://orcid.org/0000-0002-9180-2935"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Si Liu","raw_affiliation_strings":["School of Artificial Intelligence, Beihang University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5106554007"],"corresponding_institution_ids":["https://openalex.org/I82880672"],"apc_list":null,"apc_paid":null,"fwci":0.7274,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.76946548,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"9485","last_page":"9494"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9871000051498413,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.9805999994277954,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/panopticon","display_name":"Panopticon","score":0.7527779340744019},{"id":"https://openalex.org/keywords/narrative","display_name":"Narrative","score":0.7505804300308228},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6142889261245728},{"id":"https://openalex.org/keywords/diffusion","display_name":"Diffusion","score":0.5571611523628235},{"id":"https://openalex.org/keywords/ground","display_name":"Ground","score":0.48981377482414246},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.4547629952430725},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.4296887516975403},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.4114701747894287},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3478890359401703},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3404397964477539},{"id":"https://openalex.org/keywords/art","display_name":"Art","score":0.22986644506454468},{"id":"https://openalex.org/keywords/literature","display_name":"Literature","score":0.20234066247940063},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.1888534128665924},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.14493584632873535},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.14398670196533203},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.11928200721740723}],"concepts":[{"id":"https://openalex.org/C138569888","wikidata":"https://www.wikidata.org/wiki/Q828310","display_name":"Panopticon","level":3,"score":0.7527779340744019},{"id":"https://openalex.org/C199033989","wikidata":"https://www.wikidata.org/wiki/Q1318295","display_name":"Narrative","level":2,"score":0.7505804300308228},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6142889261245728},{"id":"https://openalex.org/C69357855","wikidata":"https://www.wikidata.org/wiki/Q163214","display_name":"Diffusion","level":2,"score":0.5571611523628235},{"id":"https://openalex.org/C168993435","wikidata":"https://www.wikidata.org/wiki/Q6501125","display_name":"Ground","level":2,"score":0.48981377482414246},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.4547629952430725},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.4296887516975403},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4114701747894287},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3478890359401703},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3404397964477539},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.22986644506454468},{"id":"https://openalex.org/C124952713","wikidata":"https://www.wikidata.org/wiki/Q8242","display_name":"Literature","level":1,"score":0.20234066247940063},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.1888534128665924},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.14493584632873535},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.14398670196533203},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.11928200721740723},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.0},{"id":"https://openalex.org/C2780430339","wikidata":"https://www.wikidata.org/wiki/Q10861465","display_name":"Brother","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3664647.3686836","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3686836","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W1861492603","https://openalex.org/W1903029394","https://openalex.org/W2194775991","https://openalex.org/W2302548814","https://openalex.org/W2734349601","https://openalex.org/W2981852735","https://openalex.org/W3034325957","https://openalex.org/W3092961994","https://openalex.org/W3095670406","https://openalex.org/W3108748824","https://openalex.org/W3177892185","https://openalex.org/W3178075329","https://openalex.org/W3201770677","https://openalex.org/W3216551675","https://openalex.org/W4200631575","https://openalex.org/W4212774754","https://openalex.org/W4214530390","https://openalex.org/W4288089799","https://openalex.org/W4304080713","https://openalex.org/W4311189605","https://openalex.org/W4312543911","https://openalex.org/W4312690830","https://openalex.org/W4312815172","https://openalex.org/W4312933868","https://openalex.org/W4312956471","https://openalex.org/W4315473428","https://openalex.org/W4381415952","https://openalex.org/W4382240193","https://openalex.org/W4382461696","https://openalex.org/W4385767984","https://openalex.org/W4386075493","https://openalex.org/W4386075819","https://openalex.org/W4390871886","https://openalex.org/W4390872876","https://openalex.org/W4390873429"],"related_works":["https://openalex.org/W2921107741","https://openalex.org/W2197002326","https://openalex.org/W2494728058","https://openalex.org/W3204968380","https://openalex.org/W1710116222","https://openalex.org/W2800383628","https://openalex.org/W2345320341","https://openalex.org/W4233922521","https://openalex.org/W2492288033","https://openalex.org/W2110567475"],"abstract_inverted_index":{"Panoptic":[0],"narrative":[1,20],"grounding":[2],"(PNG),":[3],"whose":[4],"core":[5],"target":[6],"is":[7],"fine-grained":[8,55,129],"image-text":[9,56,130],"alignment,":[10],"requires":[11],"a":[12,19,88,143],"panoptic":[13,32],"segmentation":[14,33,64,158],"of":[15,43,70,133],"referred":[16],"objects":[17],"given":[18],"caption.":[21],"Previous":[22],"discriminative":[23],"methods":[24],"achieve":[25,54],"only":[26],"weak":[27],"or":[28,35],"coarse-grained":[29],"alignment":[30,57,131],"by":[31],"pretraining":[34],"CLIP":[36],"model":[37],"adaptation.":[38],"Given":[39],"the":[40,67,82,109,122,128,163],"recent":[41],"progress":[42],"text-to-image":[44],"Diffusion":[45,79,110,134],"models,":[46],"several":[47],"works":[48],"have":[49],"shown":[50],"their":[51],"capability":[52,132],"to":[53,76,81,112,149],"through":[58],"cross-attention":[59],"maps":[60],"and":[61,92,120,154],"improved":[62],"general":[63],"performance.":[65,98,173],"However,":[66],"direct":[68],"use":[69],"phrase":[71,115,155],"features":[72,119,156],"as":[73],"static":[74],"prompts":[75,116],"apply":[77],"frozen":[78],"models":[80,135],"PNG":[83,164],"task":[84,90],"still":[85],"suffers":[86],"from":[87],"large":[89],"gap":[91],"insufficient":[93],"vision-language":[94],"interaction,":[95],"yielding":[96],"inferior":[97],"Therefore,":[99],"we":[100,140],"propose":[101],"an":[102],"Extractive-Injective":[103],"Phrase":[104],"Adapter":[105],"(EIPA)":[106],"bypass":[107],"within":[108],"UNet":[111],"dynamically":[113],"update":[114],"with":[117],"image":[118,153],"inject":[121],"multimodal":[123],"cues":[124],"back,":[125],"which":[126],"leverages":[127],"more":[136],"sufficiently.":[137],"In":[138],"addition,":[139],"also":[141],"design":[142],"Multi-Level":[144],"Mutual":[145],"Aggregation":[146],"(MLMA)":[147],"module":[148],"reciprocally":[150],"fuse":[151],"multi-level":[152],"for":[157],"refinement.":[159],"Extensive":[160],"experiments":[161],"on":[162],"benchmark":[165],"show":[166],"that":[167],"our":[168],"method":[169],"achieves":[170],"new":[171],"state-of-the-art":[172]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2025-12-19T19:40:27.379048","created_date":"2025-10-10T00:00:00"}