{"id":"https://openalex.org/W2943885184","doi":"https://doi.org/10.1109/tmm.2019.2915033","title":"Deep Hierarchical Encoder\u2013Decoder Network for Image Captioning","display_name":"Deep Hierarchical Encoder\u2013Decoder Network for Image Captioning","publication_year":2019,"publication_date":"2019-05-09","ids":{"openalex":"https://openalex.org/W2943885184","doi":"https://doi.org/10.1109/tmm.2019.2915033","mag":"2943885184"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2019.2915033","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2019.2915033","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102999463","display_name":"Xinyu Xiao","orcid":"https://orcid.org/0000-0003-4895-632X"},"institutions":[{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinyu Xiao","raw_affiliation_strings":["National Laboratory of Pattern Recognition, Institute of Automation, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-4895-632X","affiliations":[{"raw_affiliation_string":"National Laboratory of Pattern Recognition, Institute of Automation, Beijing, China","institution_ids":["https://openalex.org/I4210094879"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100629700","display_name":"Lingfeng Wang","orcid":"https://orcid.org/0000-0003-3707-0267"},"institutions":[{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lingfeng Wang","raw_affiliation_strings":["National Laboratory of Pattern Recognition, Institute of Automation, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3707-0267","affiliations":[{"raw_affiliation_string":"National Laboratory of Pattern Recognition, Institute of Automation, Beijing, China","institution_ids":["https://openalex.org/I4210094879"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101713583","display_name":"Kun Ding","orcid":"https://orcid.org/0000-0003-0657-1608"},"institutions":[{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kun Ding","raw_affiliation_strings":["National Laboratory of Pattern Recognition, Institute of Automation, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0657-1608","affiliations":[{"raw_affiliation_string":"National Laboratory of Pattern Recognition, Institute of Automation, Beijing, China","institution_ids":["https://openalex.org/I4210094879"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040673285","display_name":"Shiming Xiang","orcid":"https://orcid.org/0000-0002-2089-9733"},"institutions":[{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shiming Xiang","raw_affiliation_strings":["National Laboratory of Pattern Recognition, Institute of Automation, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2089-9733","affiliations":[{"raw_affiliation_string":"National Laboratory of Pattern Recognition, Institute of Automation, Beijing, China","institution_ids":["https://openalex.org/I4210094879"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100435212","display_name":"Chunhong Pan","orcid":"https://orcid.org/0000-0001-7433-4474"},"institutions":[{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chunhong Pan","raw_affiliation_strings":["National Laboratory of Pattern Recognition, Institute of Automation, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Laboratory of Pattern Recognition, Institute of Automation, Beijing, China","institution_ids":["https://openalex.org/I4210094879"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.7793,"has_fulltext":false,"cited_by_count":124,"citation_normalized_percentile":{"value":0.96036657,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"21","issue":"11","first_page":"2942","last_page":"2956"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.8800936937332153},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8739719390869141},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.6395258903503418},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.4619593620300293},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4259200096130371},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.4214904308319092},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3493187725543976},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.14263364672660828}],"concepts":[{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.8800936937332153},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8739719390869141},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.6395258903503418},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.4619593620300293},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4259200096130371},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.4214904308319092},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3493187725543976},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.14263364672660828},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2019.2915033","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2019.2915033","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6499999761581421}],"awards":[{"id":"https://openalex.org/G2282123724","display_name":"\u57ce\u5e02\u9053\u8def\u4ea4\u901a\u5927\u6570\u636e\u878d\u5408\u65b9\u6cd5\u7814\u7a76","funder_award_id":"91646207","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3069996871","display_name":null,"funder_award_id":"61773377","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3423164306","display_name":null,"funder_award_id":"61573352","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6647621006","display_name":null,"funder_award_id":"L172053","funder_id":"https://openalex.org/F4320322919","funder_display_name":"Natural Science Foundation of Beijing Municipality"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322919","display_name":"Natural Science Foundation of Beijing Municipality","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":115,"referenced_works":["https://openalex.org/W8316075","https://openalex.org/W1514535095","https://openalex.org/W1527575280","https://openalex.org/W1686810756","https://openalex.org/W1687846465","https://openalex.org/W1810943226","https://openalex.org/W1811254738","https://openalex.org/W1858383477","https://openalex.org/W1861492603","https://openalex.org/W1895577753","https://openalex.org/W1897761818","https://openalex.org/W1902237438","https://openalex.org/W1905882502","https://openalex.org/W1956340063","https://openalex.org/W1957706851","https://openalex.org/W1969616664","https://openalex.org/W2064675550","https://openalex.org/W2097117768","https://openalex.org/W2105103432","https://openalex.org/W2108598243","https://openalex.org/W2112912048","https://openalex.org/W2117539524","https://openalex.org/W2118434577","https://openalex.org/W2119775030","https://openalex.org/W2121863487","https://openalex.org/W2123301721","https://openalex.org/W2130942839","https://openalex.org/W2133564696","https://openalex.org/W2135726080","https://openalex.org/W2142900973","https://openalex.org/W2143449221","https://openalex.org/W2149172860","https://openalex.org/W2154652894","https://openalex.org/W2157331557","https://openalex.org/W2163605009","https://openalex.org/W2171361956","https://openalex.org/W2176263492","https://openalex.org/W2185175083","https://openalex.org/W2194775991","https://openalex.org/W2294130536","https://openalex.org/W2296385829","https://openalex.org/W2300739738","https://openalex.org/W2302086703","https://openalex.org/W2339652278","https://openalex.org/W2402302915","https://openalex.org/W2412782625","https://openalex.org/W2467570466","https://openalex.org/W2487852963","https://openalex.org/W2508429489","https://openalex.org/W2525778437","https://openalex.org/W2531380228","https://openalex.org/W2550553598","https://openalex.org/W2552161745","https://openalex.org/W2558834163","https://openalex.org/W2560645892","https://openalex.org/W2561930824","https://openalex.org/W2575842049","https://openalex.org/W2585986179","https://openalex.org/W2739107216","https://openalex.org/W2745461083","https://openalex.org/W2752461516","https://openalex.org/W2754927243","https://openalex.org/W2766447205","https://openalex.org/W2788527657","https://openalex.org/W2949376505","https://openalex.org/W2962760898","https://openalex.org/W2962835968","https://openalex.org/W2963040148","https://openalex.org/W2963084599","https://openalex.org/W2963201326","https://openalex.org/W2963248296","https://openalex.org/W2963360726","https://openalex.org/W2963630207","https://openalex.org/W2963899908","https://openalex.org/W2963918688","https://openalex.org/W2963991316","https://openalex.org/W2964018924","https://openalex.org/W2964049455","https://openalex.org/W2964308564","https://openalex.org/W2964350391","https://openalex.org/W3103022576","https://openalex.org/W4214717370","https://openalex.org/W4299801216","https://openalex.org/W6600334730","https://openalex.org/W6630875275","https://openalex.org/W6631516269","https://openalex.org/W6637306801","https://openalex.org/W6637373629","https://openalex.org/W6638273328","https://openalex.org/W6638742206","https://openalex.org/W6639102338","https://openalex.org/W6639118148","https://openalex.org/W6676647902","https://openalex.org/W6677994088","https://openalex.org/W6678262379","https://openalex.org/W6679434410","https://openalex.org/W6679436768","https://openalex.org/W6681184217","https://openalex.org/W6682086108","https://openalex.org/W6682631176","https://openalex.org/W6684191040","https://openalex.org/W6685183736","https://openalex.org/W6685230081","https://openalex.org/W6685322675","https://openalex.org/W6694260854","https://openalex.org/W6712909570","https://openalex.org/W6714414533","https://openalex.org/W6725287029","https://openalex.org/W6727690538","https://openalex.org/W6729046916","https://openalex.org/W6733236485","https://openalex.org/W6743895910","https://openalex.org/W6744684495","https://openalex.org/W6748492532","https://openalex.org/W6898505805"],"related_works":["https://openalex.org/W4210416330","https://openalex.org/W2775506363","https://openalex.org/W3164229987","https://openalex.org/W3215212336","https://openalex.org/W4290852288","https://openalex.org/W3217388757","https://openalex.org/W3122720459","https://openalex.org/W4298897568","https://openalex.org/W1938708284","https://openalex.org/W4380190185"],"abstract_inverted_index":{"Encoder-decoder":[0],"models":[1,275],"have":[2],"been":[3],"widely":[4],"used":[5],"in":[6,111,117,152],"image":[7,42,72,173,247,286],"captioning,":[8,73],"and":[9,29,87,109,125,175,194,203,230,250,282,288],"most":[10],"of":[11,24,41,52,85,93,99,107,120,127,143,148,160,168,172,179,183,185,206,256],"them":[12],"are":[13,31,122,188],"designed":[14],"via":[15],"single":[16],"long":[17],"short":[18],"term":[19],"memory":[20],"(LSTM).":[21],"The":[22,135,146],"capacity":[23,98],"single-layer":[25],"network,":[26],"whose":[27],"encoder":[28,86,142],"decoder":[30],"integrated":[32],"together,":[33],"is":[34,69,79,91,130,138,154,214],"limited":[35],"for":[36,71],"such":[37],"a":[38,63,75,217],"complex":[39],"task":[40],"captioning.":[43],"Moreover,":[44],"how":[45],"to":[46,55,81,102,132,155,190,221,248],"effectively":[47],"increase":[48],"the":[49,83,96,141,149,157,166,192,198,204,207,212,223,228,231,234,241,254,257,294],"\u201cvertical":[50],"depth\u201d":[51],"encoder-decoder":[53,67,153],"remains":[54],"be":[56],"solved.":[57],"To":[58],"deal":[59],"with":[60],"these":[61,128],"problems,":[62],"novel":[64],"deep":[65,76,100],"hierarchical":[66,77,262],"network":[68],"proposed":[70],"where":[74],"structure":[78],"explored":[80],"separate":[82],"functions":[84],"decoder.":[88],"This":[89],"model":[90,187,245,271],"capable":[92],"efficiently":[94],"exerting":[95],"representation":[97],"networks":[101],"fuse":[103],"high":[104],"level":[105],"semantics":[106],"vision":[108],"language":[110],"generating":[112],"captions.":[113],"Specifically,":[114],"visual":[115,199],"representations":[116],"top":[118],"levels":[119,129],"abstraction":[121],"simultaneously":[123],"considered,":[124],"each":[126],"associated":[131],"one":[133],"LSTM.":[134,162],"bottom-most":[136],"LSTM":[137,209],"applied":[139],"as":[140],"textual":[144,201],"inputs.":[145],"application":[147],"middle":[150,208],"layer":[151],"enhance":[156],"decoding":[158],"ability":[159],"top-most":[161],"Furthermore,":[163],"depending":[164],"on":[165,276],"introduction":[167],"semantic":[169],"enhancement":[170],"module":[171,178],"feature":[174],"distribution":[176],"combine":[177],"text":[180],"feature,":[181],"variants":[182],"architectures":[184],"our":[186,244,270,291,301],"constructed":[189],"explore":[191],"impacts":[193],"mutual":[195],"interactions":[196],"among":[197],"representation,":[200],"representations,":[202],"output":[205],"layer.":[210],"Particularly,":[211],"framework":[213],"training":[215,229],"under":[216],"reinforcement":[218],"learning":[219],"method":[220,292,302],"address":[222],"exposure":[224],"bias":[225],"problem":[226],"between":[227],"testing":[232],"by":[233],"policy":[235],"gradient":[236],"optimization.":[237],"Qualitative":[238],"analyses":[239],"indicate":[240],"process":[242],"that":[243,269],"\u201ctranslates\u201d":[246],"sentence":[249],"further":[251],"visualization":[252],"presents":[253],"evolution":[255],"hidden":[258],"states":[259],"from":[260],"different":[261],"LSTMs":[263],"over":[264],"time.":[265],"Extensive":[266],"experiments":[267],"demonstrate":[268],"outperforms":[272],"current":[273],"state-of-the-art":[274],"three":[277],"benchmark":[278],"datasets:":[279],"Flickr8K,":[280],"Flickr30K,":[281],"MSCOCO.":[283],"On":[284,297],"both":[285],"captioning":[287,299],"retrieval":[289],"tasks,":[290],"achieves":[293,304],"best":[295],"results.":[296],"MSCOCO":[298],"Leaderboard,":[300],"also":[303],"superior":[305],"performance.":[306]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":21},{"year":2024,"cited_by_count":26},{"year":2023,"cited_by_count":26},{"year":2022,"cited_by_count":21},{"year":2021,"cited_by_count":11},{"year":2020,"cited_by_count":12},{"year":2019,"cited_by_count":3}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}