{"id":"https://openalex.org/W3135254306","doi":"https://doi.org/10.1109/tmm.2021.3060948","title":"Unpaired Image Captioning With semantic-Constrained Self-Learning","display_name":"Unpaired Image Captioning With semantic-Constrained Self-Learning","publication_year":2021,"publication_date":"2021-02-24","ids":{"openalex":"https://openalex.org/W3135254306","doi":"https://doi.org/10.1109/tmm.2021.3060948","mag":"3135254306"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2021.3060948","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2021.3060948","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033097783","display_name":"Huixia Ben","orcid":"https://orcid.org/0000-0001-7946-8199"},"institutions":[{"id":"https://openalex.org/I16365422","display_name":"Hefei University of Technology","ror":"https://ror.org/02czkny70","country_code":"CN","type":"education","lineage":["https://openalex.org/I16365422"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huixia Ben","raw_affiliation_strings":["School of Computer and Information, Hefei University of Technology, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0001-7946-8199","affiliations":[{"raw_affiliation_string":"School of Computer and Information, Hefei University of Technology, Hefei, China","institution_ids":["https://openalex.org/I16365422"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085403640","display_name":"Yingwei Pan","orcid":"https://orcid.org/0000-0002-4344-8898"},"institutions":[{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yingwei Pan","raw_affiliation_strings":["CV laboratory, JD AI Research, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-4344-8898","affiliations":[{"raw_affiliation_string":"CV laboratory, JD AI Research, Beijing, China","institution_ids":["https://openalex.org/I4210103986"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041154840","display_name":"Yehao Li","orcid":"https://orcid.org/0000-0002-9603-1113"},"institutions":[{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yehao Li","raw_affiliation_strings":["CV laboratory, JD AI Research, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9603-1113","affiliations":[{"raw_affiliation_string":"CV laboratory, JD AI Research, Beijing, China","institution_ids":["https://openalex.org/I4210103986"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088760097","display_name":"Ting Yao","orcid":"https://orcid.org/0000-0001-7587-101X"},"institutions":[{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ting Yao","raw_affiliation_strings":["CV laboratory, JD AI Research, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-7587-101X","affiliations":[{"raw_affiliation_string":"CV laboratory, JD AI Research, Beijing, China","institution_ids":["https://openalex.org/I4210103986"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051332325","display_name":"Richang Hong","orcid":"https://orcid.org/0000-0001-5461-3986"},"institutions":[{"id":"https://openalex.org/I16365422","display_name":"Hefei University of Technology","ror":"https://ror.org/02czkny70","country_code":"CN","type":"education","lineage":["https://openalex.org/I16365422"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Richang Hong","raw_affiliation_strings":["School of Computer and Information, Hefei University of Technology, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0001-5461-3986","affiliations":[{"raw_affiliation_string":"School of Computer and Information, Hefei University of Technology, Hefei, China","institution_ids":["https://openalex.org/I16365422"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100377147","display_name":"Meng Wang","orcid":"https://orcid.org/0000-0002-3094-7735"},"institutions":[{"id":"https://openalex.org/I16365422","display_name":"Hefei University of Technology","ror":"https://ror.org/02czkny70","country_code":"CN","type":"education","lineage":["https://openalex.org/I16365422"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Meng Wang","raw_affiliation_strings":["School of Computer and Information, Hefei University of Technology, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0002-3094-7735","affiliations":[{"raw_affiliation_string":"School of Computer and Information, Hefei University of Technology, Hefei, China","institution_ids":["https://openalex.org/I16365422"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017597537","display_name":"Tao Mei","orcid":"https://orcid.org/0000-0003-2497-7732"},"institutions":[{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tao Mei","raw_affiliation_strings":["CV laboratory, JD AI Research, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-2497-7732","affiliations":[{"raw_affiliation_string":"CV laboratory, JD AI Research, Beijing, China","institution_ids":["https://openalex.org/I4210103986"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.1456,"has_fulltext":false,"cited_by_count":49,"citation_normalized_percentile":{"value":0.95302927,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":99},"biblio":{"volume":"24","issue":null,"first_page":"904","last_page":"916"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9850999712944031,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.93821781873703},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.798612117767334},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.6154300570487976},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5908746123313904},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.5544956922531128},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.495839923620224},{"id":"https://openalex.org/keywords/semantic-similarity","display_name":"Semantic similarity","score":0.4793992042541504},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.4199106693267822},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3201857805252075}],"concepts":[{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.93821781873703},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.798612117767334},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.6154300570487976},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5908746123313904},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.5544956922531128},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.495839923620224},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.4793992042541504},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.4199106693267822},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3201857805252075}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2021.3060948","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2021.3060948","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10","score":0.7300000190734863}],"awards":[{"id":"https://openalex.org/G1829462488","display_name":"\u8de8\u5a92\u4f53\u667a\u80fd\u95ee\u7b54\u548c\u63a8\u7406\u5173\u952e\u7406\u8bba\u4e0e\u65b9\u6cd5\u7814\u7a76","funder_award_id":"61932009","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2351125877","display_name":null,"funder_award_id":"61732007","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6151690139","display_name":null,"funder_award_id":"62020106007","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W639708223","https://openalex.org/W1773149199","https://openalex.org/W1895577753","https://openalex.org/W1905882502","https://openalex.org/W1931639407","https://openalex.org/W1939575207","https://openalex.org/W1947481528","https://openalex.org/W2099471712","https://openalex.org/W2117539524","https://openalex.org/W2122922389","https://openalex.org/W2133564696","https://openalex.org/W2194775991","https://openalex.org/W2277195237","https://openalex.org/W2302086703","https://openalex.org/W2418300416","https://openalex.org/W2552161745","https://openalex.org/W2735553491","https://openalex.org/W2739107216","https://openalex.org/W2745461083","https://openalex.org/W2886641317","https://openalex.org/W2890531016","https://openalex.org/W2913618459","https://openalex.org/W2938603906","https://openalex.org/W2943885184","https://openalex.org/W2953461088","https://openalex.org/W2954841306","https://openalex.org/W2963048642","https://openalex.org/W2963055199","https://openalex.org/W2963084599","https://openalex.org/W2963175879","https://openalex.org/W2963743213","https://openalex.org/W2963877622","https://openalex.org/W2964018924","https://openalex.org/W2974212192","https://openalex.org/W2979747405","https://openalex.org/W2981352610","https://openalex.org/W2981578638","https://openalex.org/W2981994675","https://openalex.org/W2982139784","https://openalex.org/W2983141445","https://openalex.org/W2986670728","https://openalex.org/W2990019157","https://openalex.org/W2990307191","https://openalex.org/W2992478697","https://openalex.org/W3035284526","https://openalex.org/W6679434410","https://openalex.org/W6679436768","https://openalex.org/W6729046916","https://openalex.org/W6745388339","https://openalex.org/W6745740328"],"related_works":["https://openalex.org/W2903179935","https://openalex.org/W159132833","https://openalex.org/W4385805088","https://openalex.org/W3209355071","https://openalex.org/W4317939313","https://openalex.org/W3216250699","https://openalex.org/W284732977","https://openalex.org/W2068788685","https://openalex.org/W3078371441","https://openalex.org/W2572461156"],"abstract_inverted_index":{"Image":[0],"captioning":[1,29],"has":[2],"been":[3],"an":[4,46,52,180,185],"emerging":[5],"and":[6,22,59,72,84,119,184,202,219],"fast-developing":[7],"research":[8],"topic.":[9],"Nevertheless,":[10],"most":[11],"existing":[12],"works":[13],"heavily":[14],"rely":[15],"on":[16,216,240],"large":[17],"amounts":[18],"of":[19,28,65,146,194,206,226,238],"image-sentence":[20,78],"pairs":[21,79],"therefore":[23],"hinder":[24],"the":[25,31,86,89,99,103,112,116,120,135,139,144,147,164,175,192,195,199,204,224,232],"practical":[26],"applications":[27],"in":[30,102,150,198],"wild.":[32],"In":[33],"this":[34],"paper,":[35],"we":[36,230],"present":[37],"a":[38,81,125,156],"novel":[39],"Semantic-Constrained":[40],"Self-learning":[41],"(SCS)":[42],"framework":[43],"that":[44,105],"explores":[45],"iterative":[47],"self-learning":[48],"strategy":[49],"to":[50,110,133,162,174,190],"learn":[51],"image":[53,58,118,177,247],"captioner":[54,73,83,87,154],"with":[55,88,138,172],"only":[56],"unpaired":[57,221,246],"text":[60],"data.":[61],"Technically,":[62],"SCS":[63],"consists":[64],"two":[66],"stages,":[67],"i.e.,":[68],"pseudo":[69,90,130],"pair":[70,131],"generation":[71,132,205],"re-training,":[74,155],"iteratively":[75],"producing":[76],"\"pseudo\"":[77],"via":[80,142],"pre-trained":[82],"re-training":[85],"pairs,":[91],"respectively.":[92,213],"Particularly,":[93],"both":[94,217],"stages":[95],"are":[96,188],"guided":[97],"by":[98],"recognized":[100,140],"objects":[101,141,149,197],"image,":[104],"act":[106],"as":[107],"semantic":[108,113,166],"constraint":[109],"strengthen":[111],"alignment":[114],"between":[115],"input":[117,176],"output":[121,151,200],"sentence.":[122,152],"We":[123],"leverage":[124],"semantic-constrained":[126],"beam":[127],"search":[128],"for":[129,245],"regularize":[134],"decoding":[136],"process":[137],"forcing":[143],"inclusion/exclusion":[145],"recognized/irrelevant":[148],"For":[153],"self-supervised":[157],"triplet":[158],"loss":[159],"is":[160],"utilized":[161],"preserve":[163],"relative":[165],"similarity":[167],"ordering":[168],"among":[169],"generated":[170],"sentences":[171,209],"regard":[173],"triplets.":[178],"Moreover,":[179],"object":[181],"inclusion":[182,193],"reward":[183,187],"adversarial":[186],"adopted":[189],"encourage":[191],"predicted":[196],"sentence":[201],"pursue":[203],"more":[207],"realistic":[208],"during":[210],"self-critical":[211],"training,":[212],"Experiments":[214],"conducted":[215],"dependent":[218],"independent":[220],"data":[222],"validate":[223],"superiority":[225],"SCS.":[227],"More":[228],"remarkably,":[229],"obtain":[231],"best":[233],"published":[234],"CIDEr":[235],"score":[236],"to-date":[237],"74.7\\%":[239],"COCO":[241],"Karpathy":[242],"test":[243],"split":[244],"captioning.":[248]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":13},{"year":2023,"cited_by_count":15},{"year":2022,"cited_by_count":14},{"year":2021,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}