{"id":"https://openalex.org/W2962861647","doi":"https://doi.org/10.1109/tpami.2019.2894139","title":"Hierarchical LSTMs with Adaptive Attention for Visual Captioning","display_name":"Hierarchical LSTMs with Adaptive Attention for Visual Captioning","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2962861647","doi":"https://doi.org/10.1109/tpami.2019.2894139","mag":"2962861647","pmid":"https://pubmed.ncbi.nlm.nih.gov/30668467"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2019.2894139","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2019.2894139","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5066645546","display_name":"Lianli Gao","orcid":"https://orcid.org/0000-0002-2522-6394"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lianli Gao","raw_affiliation_strings":["Future Media Center and School of Computer Science and Engineering, The University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Future Media Center and School of Computer Science and Engineering, The University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101598763","display_name":"Xiangpeng Li","orcid":"https://orcid.org/0000-0001-5350-5780"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiangpeng Li","raw_affiliation_strings":["Future Media Center and School of Computer Science and Engineering, The University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Future Media Center and School of Computer Science and Engineering, The University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036987388","display_name":"Jingkuan Song","orcid":"https://orcid.org/0000-0002-2549-8322"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingkuan Song","raw_affiliation_strings":["Future Media Center and School of Computer Science and Engineering, The University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Future Media Center and School of Computer Science and Engineering, The University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052993469","display_name":"Heng Tao Shen","orcid":"https://orcid.org/0000-0002-2999-2088"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Heng Tao Shen","raw_affiliation_strings":["Future Media Center and School of Computer Science and Engineering, The University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Future Media Center and School of Computer Science and Engineering, The University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I150229711"],"apc_list":null,"apc_paid":null,"fwci":20.0491,"has_fulltext":false,"cited_by_count":252,"citation_normalized_percentile":{"value":0.99461911,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"42","issue":"5","first_page":"1","last_page":"1"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.9836599826812744},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8587175607681274},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6047062277793884},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5919432640075684},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5490720868110657},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5348070859909058},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.47088250517845154},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.46558988094329834},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.4615400433540344},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.4607163071632385},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.42424100637435913},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4189051389694214},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.38932931423187256},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.32714056968688965},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.306327223777771}],"concepts":[{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.9836599826812744},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8587175607681274},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6047062277793884},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5919432640075684},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5490720868110657},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5348070859909058},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.47088250517845154},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.46558988094329834},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.4615400433540344},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.4607163071632385},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.42424100637435913},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4189051389694214},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.38932931423187256},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.32714056968688965},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.306327223777771},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2019.2894139","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2019.2894139","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:30668467","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/30668467","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":94,"referenced_works":["https://openalex.org/W6908809","https://openalex.org/W639708223","https://openalex.org/W877909479","https://openalex.org/W1514535095","https://openalex.org/W1522734439","https://openalex.org/W1572567476","https://openalex.org/W1573040851","https://openalex.org/W1586939924","https://openalex.org/W1686810756","https://openalex.org/W1895577753","https://openalex.org/W1902237438","https://openalex.org/W1905882502","https://openalex.org/W1923211482","https://openalex.org/W1956340063","https://openalex.org/W2078238240","https://openalex.org/W2080672747","https://openalex.org/W2097117768","https://openalex.org/W2101105183","https://openalex.org/W2107878631","https://openalex.org/W2112912048","https://openalex.org/W2123301721","https://openalex.org/W2133459682","https://openalex.org/W2139501017","https://openalex.org/W2154652894","https://openalex.org/W2156303437","https://openalex.org/W2159505618","https://openalex.org/W2163605009","https://openalex.org/W2164290393","https://openalex.org/W2176263492","https://openalex.org/W2185175083","https://openalex.org/W2190406907","https://openalex.org/W2194775991","https://openalex.org/W2277195237","https://openalex.org/W2302086703","https://openalex.org/W2425121537","https://openalex.org/W2506483933","https://openalex.org/W2507009361","https://openalex.org/W2513281263","https://openalex.org/W2514167171","https://openalex.org/W2523993696","https://openalex.org/W2550553598","https://openalex.org/W2552161745","https://openalex.org/W2556388456","https://openalex.org/W2558834163","https://openalex.org/W2560346187","https://openalex.org/W2565656701","https://openalex.org/W2575842049","https://openalex.org/W2584992898","https://openalex.org/W2607151106","https://openalex.org/W2613718673","https://openalex.org/W2737260104","https://openalex.org/W2739107216","https://openalex.org/W2745461083","https://openalex.org/W2751445731","https://openalex.org/W2752047430","https://openalex.org/W2786585376","https://openalex.org/W2887712318","https://openalex.org/W2914699769","https://openalex.org/W2949376505","https://openalex.org/W2962958773","https://openalex.org/W2963084599","https://openalex.org/W2963091558","https://openalex.org/W2963248296","https://openalex.org/W2963448089","https://openalex.org/W2963576560","https://openalex.org/W2963843052","https://openalex.org/W2963954913","https://openalex.org/W2964049455","https://openalex.org/W2964165364","https://openalex.org/W2964241990","https://openalex.org/W3099677434","https://openalex.org/W3099884890","https://openalex.org/W3103022576","https://openalex.org/W4294650876","https://openalex.org/W6600284362","https://openalex.org/W6620707391","https://openalex.org/W6623995992","https://openalex.org/W6630875275","https://openalex.org/W6634126550","https://openalex.org/W6637373629","https://openalex.org/W6676647902","https://openalex.org/W6678262379","https://openalex.org/W6682631176","https://openalex.org/W6682864246","https://openalex.org/W6683078286","https://openalex.org/W6684090549","https://openalex.org/W6684191040","https://openalex.org/W6685322675","https://openalex.org/W6687037952","https://openalex.org/W6724944384","https://openalex.org/W6725318829","https://openalex.org/W6729046916","https://openalex.org/W6743726175","https://openalex.org/W6898505805"],"related_works":["https://openalex.org/W4210416330","https://openalex.org/W3088136942","https://openalex.org/W2949362007","https://openalex.org/W2775506363","https://openalex.org/W4290852288","https://openalex.org/W4388893791","https://openalex.org/W4283207562","https://openalex.org/W3217195652","https://openalex.org/W3009270862","https://openalex.org/W2944691285"],"abstract_inverted_index":{"Recent":[0],"progress":[1],"has":[2],"been":[3],"made":[4],"in":[5,254],"using":[6,49],"attention":[7,21,60,102,119,134],"based":[8],"encoder-decoder":[9],"framework":[10,113],"for":[11,105,120,136,187,236],"image":[12,106,223],"and":[13,33,35,67,107,163,182,200,222],"video":[14,108,191,221],"captioning.":[15,74,109,192],"Most":[16],"existing":[17],"decoders":[18],"apply":[19,201],"the":[20,69,76,111,115,128,132,142,146,170,175,188,204,209,233,239,255],"mechanism":[22,61],"to":[23,126,139,156,168,203],"every":[24],"generated":[25],"word":[26],"including":[27],"both":[28,159,220,243],"visual":[29,55,73,85,143,161],"words":[30,37,44,64],"(e.g.,":[31,38],"\"gun\"":[32],"\"shooting\")":[34],"non-visual":[36,43,63],"\"the\",":[39],"\"a\").":[40],"However,":[41],"these":[42,93],"can":[45],"be":[46],"easily":[47],"predicted":[48],"natural":[50],"language":[51,147,165],"model":[52,177],"without":[53],"considering":[54],"signals":[56],"or":[57,117,124,145],"attention.":[58],"Imposing":[59],"on":[62,141,219,242],"could":[65],"mislead":[66],"decrease":[68],"overall":[70],"performance":[71,235],"of":[72,78,84,190,211,238,247],"Furthermore,":[75],"hierarchy":[77],"LSTMs":[79,153],"enables":[80],"more":[81],"complex":[82],"representation":[83],"data,":[86],"capturing":[87],"information":[88,144,162,167],"at":[89],"different":[90],"scales.":[91],"Considering":[92],"issues,":[94],"we":[95,183,194,215],"propose":[96],"a":[97,151,179],"hierarchical":[98,152],"LSTM":[99],"with":[100],"adaptive":[101,133],"(hLSTMat)":[103],"approach":[104,231],"Specifically,":[110],"proposed":[112,213],"utilizes":[114],"spatial":[116],"temporal":[118],"selecting":[121],"specific":[122],"regions":[123],"frames":[125],"predict":[127],"related":[129],"words,":[130],"while":[131],"is":[135,154,250],"deciding":[137],"whether":[138],"depend":[140],"context":[148,166],"information.":[149],"Also,":[150],"designed":[155],"simultaneously":[157],"consider":[158],"low-level":[160],"high-level":[164],"support":[169],"caption":[171],"generation.":[172],"We":[173],"design":[174],"hLSTMat":[176],"as":[178],"general":[180],"framework,":[181,214],"first":[184],"instantiate":[185,196],"it":[186,199,202],"task":[189],"Then,":[193],"further":[195],"our":[197,212,217,230],"hLSTMarefine":[198],"imioning":[205],"task.":[206],"To":[207],"demonstrate":[208],"effectiveness":[210],"test":[216],"method":[218],"captioning":[224],"tasks.":[225,244],"Experimental":[226],"results":[227],"show":[228],"that":[229],"achieves":[232],"state-of-the-art":[234],"most":[237],"evaluation":[240],"metrics":[241],"The":[245],"effect":[246],"important":[248],"components":[249],"also":[251],"well":[252],"exploited":[253],"ablation":[256],"study.":[257]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":24},{"year":2022,"cited_by_count":41},{"year":2021,"cited_by_count":61},{"year":2020,"cited_by_count":88},{"year":2019,"cited_by_count":12}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}