{"id":"https://openalex.org/W4385571007","doi":"https://doi.org/10.18653/v1/2023.acl-long.664","title":"MultiCapCLIP: Auto-Encoding Prompts for Zero-Shot Multilingual Visual Captioning","display_name":"MultiCapCLIP: Auto-Encoding Prompts for Zero-Shot Multilingual Visual Captioning","alternate_titles":[],"publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4385571007","doi":"https://doi.org/10.18653/v1/2023.acl-long.664"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.acl-long.664","doi":"https://doi.org/10.18653/v1/2023.acl-long.664","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.acl-long.664","pdf_url":"https://aclanthology.org/2023.acl-long.664.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4385571007"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.acl-long.664.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101184146","display_name":"Bang Yang","orcid":"https://orcid.org/0000-0003-2019-0377","observed_orcids":["https://orcid.org/0000-0003-2019-0377"]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bang Yang","raw_affiliation_strings":["ADSPLAB, School of ECE, Peking University","Peng Cheng Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ADSPLAB, School of ECE, Peking University","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Peng Cheng Laboratory","institution_ids":["https://openalex.org/I4210136793"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100632772","display_name":"Fenglin Liu","orcid":"https://orcid.org/0000-0001-7715-5228","observed_orcids":["https://orcid.org/0000-0001-7715-5228"]},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Fenglin Liu","raw_affiliation_strings":["University of Oxford"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oxford","institution_ids":["https://openalex.org/I40120149"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100352418","display_name":"Xian Wu","orcid":"https://orcid.org/0000-0003-1118-9710","observed_orcids":["https://orcid.org/0000-0003-1118-9710"]},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xian Wu","raw_affiliation_strings":["Tencent Jarvis Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent Jarvis Lab","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100631216","display_name":"Yaowei Wang","orcid":"https://orcid.org/0000-0003-2197-9038","observed_orcids":["https://orcid.org/0000-0003-2197-9038","https://orcid.org/0009-0001-4531-1414"]},"institutions":[{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaowei Wang","raw_affiliation_strings":["Peng Cheng Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peng Cheng Laboratory","institution_ids":["https://openalex.org/I4210136793"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xu Sun","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xu Sun","raw_affiliation_strings":["School of Computer Science, Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5002795838","display_name":"Yuexian Zou","orcid":"https://orcid.org/0000-0001-9999-6140","observed_orcids":["https://orcid.org/0000-0001-9999-6140"]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuexian Zou","raw_affiliation_strings":["ADSPLAB, School of ECE, Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ADSPLAB, School of ECE, Peking University","institution_ids":["https://openalex.org/I20231570"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.6708,"has_fulltext":true,"cited_by_count":30,"citation_normalized_percentile":{"value":0.90346592,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"11908","last_page":"11922"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multilingual-captioning","display_name":"multilingual captioning","score":0.9750000238418579},{"id":"https://openalex.org/keywords/limited-labeled-data","display_name":"limited labeled data","score":0.6639999747276306},{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.6460000276565552},{"id":"https://openalex.org/keywords/image-captioning","display_name":"image captioning","score":0.8579999804496765},{"id":"https://openalex.org/keywords/video-captioning","display_name":"video captioning","score":0.8640000224113464},{"id":"https://openalex.org/keywords/clip","display_name":"CLIP","score":0.48100000619888306}],"concepts":[{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.9690612554550171},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8599013090133667},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6149512529373169},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.5865674614906311},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5528770685195923},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.519416332244873},{"id":"https://openalex.org/C2778344882","wikidata":"https://www.wikidata.org/wiki/Q278938","display_name":"Shot (pellet)","level":2,"score":0.5114792585372925},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.4571281969547272},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4194101393222809},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3864142894744873},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.3417116403579712},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1077924370765686},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.08995017409324646},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.07678547501564026},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2023.acl-long.664","doi":"https://doi.org/10.18653/v1/2023.acl-long.664","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.acl-long.664","pdf_url":"https://aclanthology.org/2023.acl-long.664.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2308.13218","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.13218","pdf_url":"https://arxiv.org/pdf/2308.13218","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2308.13218","doi":"https://doi.org/10.48550/arxiv.2308.13218","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2308.13218","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.acl-long.664","doi":"https://doi.org/10.18653/v1/2023.acl-long.664","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.acl-long.664","pdf_url":"https://aclanthology.org/2023.acl-long.664.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1739072399","display_name":null,"funder_award_id":"62176008","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385571007.pdf","grobid_xml":"https://content.openalex.org/works/W4385571007.grobid-xml"},"referenced_works_count":76,"referenced_works":["https://openalex.org/W639708223","https://openalex.org/W1514535095","https://openalex.org/W1889081078","https://openalex.org/W1905882502","https://openalex.org/W1956340063","https://openalex.org/W2025768430","https://openalex.org/W2101105183","https://openalex.org/W2123301721","https://openalex.org/W2154652894","https://openalex.org/W2177066871","https://openalex.org/W2183341477","https://openalex.org/W2187089797","https://openalex.org/W2194775991","https://openalex.org/W2345720230","https://openalex.org/W2425121537","https://openalex.org/W2506483933","https://openalex.org/W2626778328","https://openalex.org/W2904849495","https://openalex.org/W2908510526","https://openalex.org/W2929321571","https://openalex.org/W2963341956","https://openalex.org/W2963743213","https://openalex.org/W2963909453","https://openalex.org/W2969295463","https://openalex.org/W2989322838","https://openalex.org/W2992478697","https://openalex.org/W3003478339","https://openalex.org/W3030163527","https://openalex.org/W3034493371","https://openalex.org/W3037932933","https://openalex.org/W3039060838","https://openalex.org/W3041439482","https://openalex.org/W3086385316","https://openalex.org/W3091588028","https://openalex.org/W3094502228","https://openalex.org/W3103237788","https://openalex.org/W3110014757","https://openalex.org/W3119786062","https://openalex.org/W3126337491","https://openalex.org/W3135254306","https://openalex.org/W3135367836","https://openalex.org/W3152980002","https://openalex.org/W3160190143","https://openalex.org/W3166573884","https://openalex.org/W3176689360","https://openalex.org/W3204689852","https://openalex.org/W3214040319","https://openalex.org/W3217340782","https://openalex.org/W3217578129","https://openalex.org/W4214611271","https://openalex.org/W4225620527","https://openalex.org/W4226352076","https://openalex.org/W4285606530","https://openalex.org/W4287551944","https://openalex.org/W4288043238","https://openalex.org/W4293567540","https://openalex.org/W4297808394","https://openalex.org/W4308023325","https://openalex.org/W4312463400","https://openalex.org/W4312535570","https://openalex.org/W4312712450","https://openalex.org/W4312810944","https://openalex.org/W4312877428","https://openalex.org/W4312922092","https://openalex.org/W4312938727","https://openalex.org/W4313175608","https://openalex.org/W4320339901","https://openalex.org/W4320458302","https://openalex.org/W4323570346","https://openalex.org/W4375869190","https://openalex.org/W4385567053","https://openalex.org/W4385572615","https://openalex.org/W4390889743","https://openalex.org/W4392309090","https://openalex.org/W7133217898","https://openalex.org/W7133218155"],"related_works":["https://openalex.org/W4323570346","https://openalex.org/W4387968008","https://openalex.org/W4411446509","https://openalex.org/W4408097591","https://openalex.org/W4417443965","https://openalex.org/W4309134658","https://openalex.org/W4292828275","https://openalex.org/W4386554289","https://openalex.org/W3214040319","https://openalex.org/W4385570974"],"abstract_inverted_index":{"Supervised":[0],"visual":[1,65,129,143],"captioning":[2,151],"models":[3],"typically":[4],"require":[5],"a":[6,17,55,120],"large":[7],"scale":[8],"of":[9,77,105,167,186],"images":[10],"or":[11],"videos":[12],"paired":[13],"with":[14,48,170],"descriptions":[15],"in":[16,119,184],"specific":[18],"language":[19],"(i.e.,":[20,158],"the":[21,49,80,101,110,123,136,141,165],"vision-caption":[22,75],"pairs)":[23],"for":[24,35,67,88],"training.However,":[25],"collecting":[26],"and":[27,33,38,70,149,155,162,173,180,188],"labeling":[28],"large-scale":[29],"datasets":[30],"is":[31,192],"time-consuming":[32],"expensive":[34],"many":[36],"scenarios":[37,69],"languages.Therefore,":[39],"sufficient":[40],"labeled":[41,74],"pairs":[42,76],"are":[43],"usually":[44],"not":[45],"available.To":[46],"deal":[47],"label":[50],"shortage":[51],"problem,":[52],"we":[53],"present":[54],"simple":[56],"yet":[57],"effective":[58],"zero-shot":[59,172],"approach":[60],"Mul-tiCapCLIP":[61],"that":[62,99],"can":[63],"generate":[64,140],"captions":[66,118],"different":[68],"languages":[71,157],"without":[72],"any":[73],"downstream":[78],"datasets.In":[79],"training":[81],"stage,":[82,125],"MultiCapCLIP":[83,126],"only":[84],"requires":[85],"text":[86],"data":[87,130],"input.Then":[89],"it":[90],"conducts":[91],"two":[92],"main":[93],"steps:":[94],"1)":[95],"retrieving":[96],"concept":[97,137],"prompts":[98,111,138],"preserve":[100],"corresponding":[102],"domain":[103],"knowledge":[104],"new":[106],"scenarios;":[107],"2)":[108],"autoencoding":[109],"to":[112,116,134,139],"learn":[113],"writing":[114],"styles":[115],"output":[117],"desired":[121],"language.In":[122],"testing":[124],"instead":[127],"takes":[128],"as":[131],"input":[132],"directly":[133],"retrieve":[135],"final":[142],"descriptions.The":[144],"extensive":[145],"experiments":[146],"on":[147],"image":[148],"video":[150],"across":[152],"four":[153,156],"benchmarks":[154],"English,":[159],"Chinese,":[160],"German,":[161],"French)":[163],"confirm":[164],"effectiveness":[166],"our":[168,176],"approach.Compared":[169],"state-of-theart":[171],"weakly-supervised":[174],"methods,":[175],"method":[177],"achieves":[178],"4.8%":[179],"21.5%":[181],"absolute":[182],"improvements":[183],"terms":[185],"BLEU@4":[187],"CIDEr":[189],"metrics.Our":[190],"code":[191],"available":[193],"at":[194],"https:":[195],"//github.com/yangbang18/MultiCapCLIP.":[196]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4385571007","counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2023-08-05T00:00:00"}
