{"id":"https://openalex.org/W4385486374","doi":"https://doi.org/10.1109/taslp.2023.3301210","title":"Layer-Level Progressive Transformer With Modality Difference Awareness for Multi-Modal Neural Machine Translation","display_name":"Layer-Level Progressive Transformer With Modality Difference Awareness for Multi-Modal Neural Machine Translation","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4385486374","doi":"https://doi.org/10.1109/taslp.2023.3301210"},"language":"en","primary_location":{"id":"doi:10.1109/taslp.2023.3301210","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2023.3301210","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103004657","display_name":"Junjun Guo","orcid":"https://orcid.org/0000-0002-3522-7120"},"institutions":[{"id":"https://openalex.org/I10660446","display_name":"Kunming University of Science and Technology","ror":"https://ror.org/00xyeez13","country_code":"CN","type":"education","lineage":["https://openalex.org/I10660446"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junjun Guo","raw_affiliation_strings":["Faculty of Information Engineering and Automation, Kunming University of Science and Technology, Kunming, China","Faculty of Information Engineering and Automation, Kunming University of Science and Technology, kunming, China","Yunnan Key Laboratory of Artificial Intelligence, Kunming University of Science and Technology, kunming, China"],"raw_orcid":"https://orcid.org/0000-0002-3522-7120","affiliations":[{"raw_affiliation_string":"Faculty of Information Engineering and Automation, Kunming University of Science and Technology, Kunming, China","institution_ids":["https://openalex.org/I10660446"]},{"raw_affiliation_string":"Faculty of Information Engineering and Automation, Kunming University of Science and Technology, kunming, China","institution_ids":["https://openalex.org/I10660446"]},{"raw_affiliation_string":"Yunnan Key Laboratory of Artificial Intelligence, Kunming University of Science and Technology, kunming, China","institution_ids":["https://openalex.org/I10660446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089505818","display_name":"Junjie Ye","orcid":"https://orcid.org/0000-0002-3391-7582"},"institutions":[{"id":"https://openalex.org/I10660446","display_name":"Kunming University of Science and Technology","ror":"https://ror.org/00xyeez13","country_code":"CN","type":"education","lineage":["https://openalex.org/I10660446"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junjie Ye","raw_affiliation_strings":["Faculty of Information Engineering and Automation, Kunming University of Science and Technology, Kunming, China","Faculty of Information Engineering and Automation, Kunming University of Science and Technology, kunming, China","Yunnan Key Laboratory of Artificial Intelligence, Kunming University of Science and Technology, kunming, China"],"raw_orcid":"https://orcid.org/0000-0002-3391-7582","affiliations":[{"raw_affiliation_string":"Faculty of Information Engineering and Automation, Kunming University of Science and Technology, Kunming, China","institution_ids":["https://openalex.org/I10660446"]},{"raw_affiliation_string":"Faculty of Information Engineering and Automation, Kunming University of Science and Technology, kunming, China","institution_ids":["https://openalex.org/I10660446"]},{"raw_affiliation_string":"Yunnan Key Laboratory of Artificial Intelligence, Kunming University of Science and Technology, kunming, China","institution_ids":["https://openalex.org/I10660446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084440288","display_name":"Yan Xiang","orcid":"https://orcid.org/0000-0002-6475-638X"},"institutions":[{"id":"https://openalex.org/I10660446","display_name":"Kunming University of Science and Technology","ror":"https://ror.org/00xyeez13","country_code":"CN","type":"education","lineage":["https://openalex.org/I10660446"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yan Xiang","raw_affiliation_strings":["Faculty of Information Engineering and Automation, Kunming University of Science and Technology, Kunming, China","Faculty of Information Engineering and Automation, Kunming University of Science and Technology, kunming, China","Yunnan Key Laboratory of Artificial Intelligence, Kunming University of Science and Technology, kunming, China"],"raw_orcid":"https://orcid.org/0000-0002-6475-638X","affiliations":[{"raw_affiliation_string":"Faculty of Information Engineering and Automation, Kunming University of Science and Technology, Kunming, China","institution_ids":["https://openalex.org/I10660446"]},{"raw_affiliation_string":"Faculty of Information Engineering and Automation, Kunming University of Science and Technology, kunming, China","institution_ids":["https://openalex.org/I10660446"]},{"raw_affiliation_string":"Yunnan Key Laboratory of Artificial Intelligence, Kunming University of Science and Technology, kunming, China","institution_ids":["https://openalex.org/I10660446"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100619287","display_name":"Zhengtao Yu","orcid":"https://orcid.org/0000-0002-4012-461X"},"institutions":[{"id":"https://openalex.org/I10660446","display_name":"Kunming University of Science and Technology","ror":"https://ror.org/00xyeez13","country_code":"CN","type":"education","lineage":["https://openalex.org/I10660446"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhengtao Yu","raw_affiliation_strings":["Faculty of Information Engineering and Automation, Kunming University of Science and Technology, Kunming, China","Faculty of Information Engineering and Automation, Kunming University of Science and Technology, kunming, China","Yunnan Key Laboratory of Artificial Intelligence, Kunming University of Science and Technology, kunming, China"],"raw_orcid":"https://orcid.org/0000-0002-4012-461X","affiliations":[{"raw_affiliation_string":"Faculty of Information Engineering and Automation, Kunming University of Science and Technology, Kunming, China","institution_ids":["https://openalex.org/I10660446"]},{"raw_affiliation_string":"Faculty of Information Engineering and Automation, Kunming University of Science and Technology, kunming, China","institution_ids":["https://openalex.org/I10660446"]},{"raw_affiliation_string":"Yunnan Key Laboratory of Artificial Intelligence, Kunming University of Science and Technology, kunming, China","institution_ids":["https://openalex.org/I10660446"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I10660446"],"apc_list":null,"apc_paid":null,"fwci":1.4403,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.8491159,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"31","issue":null,"first_page":"3015","last_page":"3026"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6572800278663635},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.6301760673522949},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.611146092414856},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.547603964805603},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5379375219345093},{"id":"https://openalex.org/keywords/semantic-gap","display_name":"Semantic gap","score":0.5200307369232178},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5154460072517395},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.49262863397598267},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.46952858567237854},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.34719985723495483},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.10340946912765503},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07684639096260071}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6572800278663635},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.6301760673522949},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.611146092414856},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.547603964805603},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5379375219345093},{"id":"https://openalex.org/C86034646","wikidata":"https://www.wikidata.org/wiki/Q474311","display_name":"Semantic gap","level":4,"score":0.5200307369232178},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5154460072517395},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.49262863397598267},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.46952858567237854},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.34719985723495483},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.10340946912765503},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07684639096260071},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C1667742","wikidata":"https://www.wikidata.org/wiki/Q10927554","display_name":"Image retrieval","level":3,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taslp.2023.3301210","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2023.3301210","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6899999976158142,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G3062682909","display_name":null,"funder_award_id":"62241604","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5714086050","display_name":"\u57fa\u4e8e\u56fe\u50cf\u3001\u6587\u672c\u7279\u5f81\u5bf9\u9f50\u878d\u5408\u7684\u591a\u6a21\u6001\u795e\u7ecf\u673a\u5668\u7ffb\u8bd1\u95ee\u9898\u7684\u7814\u7a76","funder_award_id":"61866020","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":56,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2133459682","https://openalex.org/W2194775991","https://openalex.org/W2417549359","https://openalex.org/W2513263213","https://openalex.org/W2581101319","https://openalex.org/W2593341061","https://openalex.org/W2726413184","https://openalex.org/W2765407302","https://openalex.org/W2789954303","https://openalex.org/W2887678105","https://openalex.org/W2888070626","https://openalex.org/W2889545026","https://openalex.org/W2889903020","https://openalex.org/W2891021639","https://openalex.org/W2903343986","https://openalex.org/W2934881944","https://openalex.org/W2950207430","https://openalex.org/W2950886580","https://openalex.org/W2962784628","https://openalex.org/W2963331233","https://openalex.org/W2963360627","https://openalex.org/W2963909453","https://openalex.org/W2964192290","https://openalex.org/W2980216782","https://openalex.org/W2986752547","https://openalex.org/W2992308087","https://openalex.org/W2997205428","https://openalex.org/W3006381853","https://openalex.org/W3034773362","https://openalex.org/W3034871396","https://openalex.org/W3035421790","https://openalex.org/W3083614545","https://openalex.org/W3102475290","https://openalex.org/W3106013275","https://openalex.org/W3109387650","https://openalex.org/W3114822941","https://openalex.org/W3115242847","https://openalex.org/W3119746927","https://openalex.org/W3154571463","https://openalex.org/W3163658745","https://openalex.org/W3174570731","https://openalex.org/W3176395632","https://openalex.org/W3197158004","https://openalex.org/W3216735052","https://openalex.org/W4206119104","https://openalex.org/W4206588072","https://openalex.org/W4214911019","https://openalex.org/W4297780100","https://openalex.org/W4385245566","https://openalex.org/W6729887601","https://openalex.org/W6739901393","https://openalex.org/W6745136726","https://openalex.org/W6753768231","https://openalex.org/W6760871911","https://openalex.org/W6771713106"],"related_works":["https://openalex.org/W3176018525","https://openalex.org/W2903533908","https://openalex.org/W2903810591","https://openalex.org/W627697492","https://openalex.org/W2963499882","https://openalex.org/W3026554633","https://openalex.org/W2888520903","https://openalex.org/W2890256614","https://openalex.org/W3098873988","https://openalex.org/W2949454572"],"abstract_inverted_index":{"Multi-modal":[0],"neural":[1],"machine":[2,57,180],"translation":[3,58,181],"(MNMT)":[4],"aims":[5],"to":[6,45,52,98,104,131,144,170,178],"translate":[7],"sentences":[8],"from":[9],"the":[10,14,18,31,54,62,68,100,109,133,146,186,192,196,200,216],"source":[11],"language":[12,16],"into":[13,50],"target":[15],"with":[17,77,127],"aid":[19],"of":[20,38,56,61],"corresponding":[21],"images.":[22],"Unfortunately,":[23],"there":[24,82],"is":[25,59,83,113,168],"a":[26,84,122,162],"considerable":[27,85],"modality":[28,110],"gap":[29,87,112],"between":[30,88],"semantic-related":[32],"images":[33,89],"and":[34,41,71,90,152,174,191,210],"texts":[35,51],"in":[36,138,157],"terms":[37],"data":[39],"form":[40],"semantic":[42],"expression.":[43],"How":[44],"fully":[46],"incorporate":[47],"visual":[48,70,151,173],"information":[49,148],"enhance":[53,179],"performance":[55],"one":[60],"critical":[63],"issues":[64],"for":[65],"MNMT.":[66,139],"However,":[67],"initial":[69],"textual":[72,153,175],"features":[73],"are":[74,189],"generally":[75],"extracted":[76],"their":[78],"modality-specific":[79],"models;":[80],"Consequently,":[81],"representation":[86,111],"texts.":[91],"Most":[92],"previous":[93],"MNMT":[94],"works":[95],"prefer":[96],"only":[97],"adopt":[99],"feature-level":[101],"fusion":[102,135,165],"strategies":[103],"learn":[105],"multi-modal":[106,124],"representation,":[107],"while":[108],"often":[114],"ignored.":[115],"To":[116],"this":[117,119],"end,":[118],"paper":[120],"proposes":[121],"progressive":[123,164],"Transformer":[125,159],"(ProMul-Trans)":[126],"Modality":[128],"Difference-Aware":[129],"(MDA)":[130],"address":[132],"visual-to-textual":[134],"problem":[136],"raised":[137],"We":[140,214],"first":[141],"employ":[142],"MDA":[143],"capture":[145],"modality-consistency":[147],"by":[149],"taking":[150],"representations":[154,176],"as":[155],"inputs":[156],"each":[158],"layer.":[160],"Then":[161],"layer-level":[163],"(Layer-ProFusion)":[166],"strategy":[167],"adopted":[169],"progressively":[171],"align":[172],"layer-by-layer":[177],"performance.":[182],"Experiment":[183],"results":[184,193],"on":[185,205],"Multi30k":[187],"dataset":[188],"conducted,":[190],"show":[194],"that":[195],"proposed":[197],"approach":[198],"outperforms":[199],"compared":[201],"state-of-the-art":[202],"(SOTA)":[203],"methods":[204],"English$\\rightarrow$German":[206],"(En$\\rightarrow$De),":[207],"English$\\rightarrow$French":[208],"(En$\\rightarrow$Fr)":[209],"English$\\rightarrow$Czech":[211],"(En$\\rightarrow$Cs)":[212],"tasks.":[213],"release":[215],"code":[217],"athttps://github.com/JunjieYe-MMT/HierProMul-Trans.":[218]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":3}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}