{"id":"https://openalex.org/W3213610319","doi":"https://doi.org/10.1109/tmm.2021.3122542","title":"Joint Embedding of Deep Visual and Semantic Features for Medical Image Report Generation","display_name":"Joint Embedding of Deep Visual and Semantic Features for Medical Image Report Generation","publication_year":2021,"publication_date":"2021-11-08","ids":{"openalex":"https://openalex.org/W3213610319","doi":"https://doi.org/10.1109/tmm.2021.3122542","mag":"3213610319"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2021.3122542","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2021.3122542","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029741928","display_name":"Yan Yang","orcid":"https://orcid.org/0000-0001-5598-1692"},"institutions":[{"id":"https://openalex.org/I50760025","display_name":"Hangzhou Dianzi University","ror":"https://ror.org/0576gt767","country_code":"CN","type":"education","lineage":["https://openalex.org/I50760025"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yan Yang","raw_affiliation_strings":["Key Laboratory of Complex Systems Modeling and Simulation, School of Computer Science and Technology, Hangzhou Dianzi University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-5598-1692","affiliations":[{"raw_affiliation_string":"Key Laboratory of Complex Systems Modeling and Simulation, School of Computer Science and Technology, Hangzhou Dianzi University, Hangzhou, China","institution_ids":["https://openalex.org/I50760025"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050817770","display_name":"Jun Yu","orcid":"https://orcid.org/0000-0003-1922-7283"},"institutions":[{"id":"https://openalex.org/I50760025","display_name":"Hangzhou Dianzi University","ror":"https://ror.org/0576gt767","country_code":"CN","type":"education","lineage":["https://openalex.org/I50760025"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Yu","raw_affiliation_strings":["Key Laboratory of Complex Systems Modeling and Simulation, School of Computer Science and Technology, Hangzhou Dianzi University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-1922-7283","affiliations":[{"raw_affiliation_string":"Key Laboratory of Complex Systems Modeling and Simulation, School of Computer Science and Technology, Hangzhou Dianzi University, Hangzhou, China","institution_ids":["https://openalex.org/I50760025"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042342029","display_name":"Jian Zhang","orcid":"https://orcid.org/0000-0001-6478-9192"},"institutions":[{"id":"https://openalex.org/I105126617","display_name":"Zhejiang International Studies University","ror":"https://ror.org/01vwvvq12","country_code":"CN","type":"education","lineage":["https://openalex.org/I105126617"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian Zhang","raw_affiliation_strings":["Zhejiang International Studies University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-6478-9192","affiliations":[{"raw_affiliation_string":"Zhejiang International Studies University, Hangzhou, China","institution_ids":["https://openalex.org/I105126617"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003662438","display_name":"Weidong Han","orcid":"https://orcid.org/0000-0001-7227-3671"},"institutions":[{"id":"https://openalex.org/I2802390878","display_name":"Sir Run Run Shaw Hospital","ror":"https://ror.org/00ka6rp58","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I2802390878"]},{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weidong Han","raw_affiliation_strings":["Department of Medical Oncology, Sir Run Run Shaw Hospital, College of Medicine, Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-7227-3671","affiliations":[{"raw_affiliation_string":"Department of Medical Oncology, Sir Run Run Shaw Hospital, College of Medicine, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I2802390878","https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045151493","display_name":"Hanliang Jiang","orcid":"https://orcid.org/0000-0003-0902-882X"},"institutions":[{"id":"https://openalex.org/I2802390878","display_name":"Sir Run Run Shaw Hospital","ror":"https://ror.org/00ka6rp58","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I2802390878"]},{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hanliang Jiang","raw_affiliation_strings":["Regional Medical Center for National Institute of Respiratory Diseases, Sir Run Run Shaw Hospital, College of Medicine, Zhejiang University, Hangzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Regional Medical Center for National Institute of Respiratory Diseases, Sir Run Run Shaw Hospital, College of Medicine, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I2802390878","https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028597017","display_name":"Qingming Huang","orcid":"https://orcid.org/0000-0001-7542-296X"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qingming Huang","raw_affiliation_strings":["School of Computer and Control Engineering, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-7542-296X","affiliations":[{"raw_affiliation_string":"School of Computer and Control Engineering, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5029741928"],"corresponding_institution_ids":["https://openalex.org/I50760025"],"apc_list":null,"apc_paid":null,"fwci":3.1035,"has_fulltext":false,"cited_by_count":68,"citation_normalized_percentile":{"value":0.93182588,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"25","issue":null,"first_page":"167","last_page":"178"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9943000078201294,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8351453542709351},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.7863019704818726},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.6168925762176514},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5946285128593445},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5707552433013916},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5416463017463684},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5400884747505188},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5060810446739197},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.41203010082244873},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.154660165309906}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8351453542709351},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.7863019704818726},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.6168925762176514},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5946285128593445},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5707552433013916},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5416463017463684},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5400884747505188},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5060810446739197},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.41203010082244873},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.154660165309906},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2021.3122542","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2021.3122542","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.8500000238418579,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G7252885093","display_name":null,"funder_award_id":"62176230","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":48,"referenced_works":["https://openalex.org/W68733909","https://openalex.org/W1548445631","https://openalex.org/W1895577753","https://openalex.org/W1897761818","https://openalex.org/W1956340063","https://openalex.org/W1978394996","https://openalex.org/W2027423900","https://openalex.org/W2064675550","https://openalex.org/W2101105183","https://openalex.org/W2152772232","https://openalex.org/W2194775991","https://openalex.org/W2302086703","https://openalex.org/W2334763311","https://openalex.org/W2549599535","https://openalex.org/W2604178507","https://openalex.org/W2607151106","https://openalex.org/W2770165365","https://openalex.org/W2778310824","https://openalex.org/W2890888035","https://openalex.org/W2896348597","https://openalex.org/W2903721568","https://openalex.org/W2908056713","https://openalex.org/W2942396292","https://openalex.org/W2949167374","https://openalex.org/W2950212751","https://openalex.org/W2962861647","https://openalex.org/W2962968835","https://openalex.org/W2963466845","https://openalex.org/W2963527096","https://openalex.org/W2963967185","https://openalex.org/W2974212192","https://openalex.org/W2979861699","https://openalex.org/W2987327987","https://openalex.org/W2995225687","https://openalex.org/W2997704374","https://openalex.org/W3034984754","https://openalex.org/W3098325931","https://openalex.org/W3100183449","https://openalex.org/W3104027471","https://openalex.org/W6631190155","https://openalex.org/W6632924514","https://openalex.org/W6637306801","https://openalex.org/W6676497082","https://openalex.org/W6682631176","https://openalex.org/W6747741389","https://openalex.org/W6751711410","https://openalex.org/W6756147243","https://openalex.org/W6766193724"],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W4246352526","https://openalex.org/W2121910908","https://openalex.org/W2129146436","https://openalex.org/W2032507829"],"abstract_inverted_index":{"Medical":[0],"image":[1],"report":[2,103,109,143],"generation":[3],"(MeIRG)":[4],"aims":[5],"at":[6,163],"generating":[7],"associated":[8],"diagnosis":[9,24],"descriptions":[10],"with":[11,97],"natural":[12],"language":[13],"sentences":[14],"from":[15,94],"medical":[16,33,95,102,115,124],"images,":[17],"which":[18,42],"is":[19,67],"essential":[20],"in":[21,31,47],"the":[22,48,62,83,98,142,153],"computer-aided":[23],"system.":[25],"Nevertheless,":[26],"this":[27,52],"task":[28],"remains":[29],"challenging":[30],"that":[32,60],"images":[34,96],"and":[35,74,136],"linguistic":[36],"expressions":[37],"should":[38],"be":[39],"understood":[40],"jointly":[41],"however":[43],"show":[44],"great":[45],"discrepancies":[46],"modality.":[49],"To":[50],"fill":[51],"visual-to-semantic":[53],"gap,":[54],"we":[55],"propose":[56],"a":[57,78,139],"novel":[58],"framework":[59,66],"follows":[61],"encoder-decoder":[63],"pipeline.":[64],"Our":[65],"characterized":[68],"by":[69],"encoding":[70,84],"both":[71],"deep":[72],"visual":[73,87,92],"semantic":[75,108,120],"embeddings":[76,93,121],"through":[77],"triple-branch":[79],"network":[80],"(TriNet)":[81],"during":[82],"phase.":[85],"The":[86,101,111],"attention":[88],"branch":[89,106,113],"captures":[90],"attended":[91],"soft-attention":[99],"mechanism.":[100],"(MeRP)":[104],"embedding":[105,112],"predicts":[107],"embeddings.":[110],"of":[114,122,131,156],"subject":[116],"headings":[117],"(MeSH)":[118],"obtains":[119],"related":[123],"tags":[125],"as":[126],"complementary":[127],"information.":[128],"Then,":[129],"outputs":[130],"these":[132],"branches":[133],"are":[134,161],"fused":[135],"fed":[137],"into":[138],"decoder":[140],"for":[141],"generation.":[144],"Experimental":[145],"results":[146],"on":[147],"two":[148],"benchmark":[149],"datasets":[150],"have":[151],"demonstrated":[152],"excellent":[154],"performance":[155],"our":[157],"method.":[158],"Related":[159],"codes":[160],"available":[162],"<uri":[164],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[165],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">https://github.com/yangyan22/Medical-Report-Generation-TriNet</uri>":[166],".":[167]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":30},{"year":2024,"cited_by_count":20},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":6}],"updated_date":"2026-05-07T13:39:58.223016","created_date":"2025-10-10T00:00:00"}