{"id":"https://openalex.org/W4392902988","doi":"https://doi.org/10.1109/icassp48485.2024.10446966","title":"An Explicit Multi-Modal Fusion Method for Sign Language Translation","display_name":"An Explicit Multi-Modal Fusion Method for Sign Language Translation","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4392902988","doi":"https://doi.org/10.1109/icassp48485.2024.10446966"},"language":"en","primary_location":{"id":"doi:10.1109/icassp48485.2024.10446966","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10446966","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101596161","display_name":"Cong Hu","orcid":"https://orcid.org/0000-0002-1228-3381"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cong Hu","raw_affiliation_strings":["Xiamen University,School of Informatics,Department of Artificial Intelligence","Department of Artificial Intelligence, School of Informatics, Xiamen University","Key Laboratory of Digital Protection and Intelligent Processing of Intangible Cultural Heritage of Fujian and Taiwan (Xiamen University), Ministry of Culture and Tourism"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xiamen University,School of Informatics,Department of Artificial Intelligence","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Department of Artificial Intelligence, School of Informatics, Xiamen University","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Key Laboratory of Digital Protection and Intelligent Processing of Intangible Cultural Heritage of Fujian and Taiwan (Xiamen University), Ministry of Culture and Tourism","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101417243","display_name":"Biao Fu","orcid":"https://orcid.org/0000-0002-4532-7432"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Biao Fu","raw_affiliation_strings":["Xiamen University,School of Informatics,Department of Artificial Intelligence","Department of Artificial Intelligence, School of Informatics, Xiamen University","Key Laboratory of Digital Protection and Intelligent Processing of Intangible Cultural Heritage of Fujian and Taiwan (Xiamen University), Ministry of Culture and Tourism"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xiamen University,School of Informatics,Department of Artificial Intelligence","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Department of Artificial Intelligence, School of Informatics, Xiamen University","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Key Laboratory of Digital Protection and Intelligent Processing of Intangible Cultural Heritage of Fujian and Taiwan (Xiamen University), Ministry of Culture and Tourism","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103197354","display_name":"Pei Yu","orcid":"https://orcid.org/0009-0000-8972-5208"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pei Yu","raw_affiliation_strings":["Xiamen University,School of Informatics,Department of Artificial Intelligence","Department of Artificial Intelligence, School of Informatics, Xiamen University","Key Laboratory of Digital Protection and Intelligent Processing of Intangible Cultural Heritage of Fujian and Taiwan (Xiamen University), Ministry of Culture and Tourism"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xiamen University,School of Informatics,Department of Artificial Intelligence","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Department of Artificial Intelligence, School of Informatics, Xiamen University","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Key Laboratory of Digital Protection and Intelligent Processing of Intangible Cultural Heritage of Fujian and Taiwan (Xiamen University), Ministry of Culture and Tourism","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100425299","display_name":"Liang Zhang","orcid":"https://orcid.org/0000-0003-4528-2403"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liang Zhang","raw_affiliation_strings":["Xiamen University,School of Informatics,Department of Artificial Intelligence","Department of Artificial Intelligence, School of Informatics, Xiamen University","Key Laboratory of Digital Protection and Intelligent Processing of Intangible Cultural Heritage of Fujian and Taiwan (Xiamen University), Ministry of Culture and Tourism"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xiamen University,School of Informatics,Department of Artificial Intelligence","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Department of Artificial Intelligence, School of Informatics, Xiamen University","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Key Laboratory of Digital Protection and Intelligent Processing of Intangible Cultural Heritage of Fujian and Taiwan (Xiamen University), Ministry of Culture and Tourism","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021246376","display_name":"Xiaodong Shi","orcid":"https://orcid.org/0000-0002-8163-7139"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaodong Shi","raw_affiliation_strings":["Xiamen University,School of Informatics,Department of Artificial Intelligence","Department of Artificial Intelligence, School of Informatics, Xiamen University","Key Laboratory of Digital Protection and Intelligent Processing of Intangible Cultural Heritage of Fujian and Taiwan (Xiamen University), Ministry of Culture and Tourism"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xiamen University,School of Informatics,Department of Artificial Intelligence","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Department of Artificial Intelligence, School of Informatics, Xiamen University","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Key Laboratory of Digital Protection and Intelligent Processing of Intangible Cultural Heritage of Fujian and Taiwan (Xiamen University), Ministry of Culture and Tourism","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100689058","display_name":"Yidong Chen","orcid":"https://orcid.org/0000-0002-0243-7228"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yidong Chen","raw_affiliation_strings":["Xiamen University,School of Informatics,Department of Artificial Intelligence","Department of Artificial Intelligence, School of Informatics, Xiamen University","Key Laboratory of Digital Protection and Intelligent Processing of Intangible Cultural Heritage of Fujian and Taiwan (Xiamen University), Ministry of Culture and Tourism"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xiamen University,School of Informatics,Department of Artificial Intelligence","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Department of Artificial Intelligence, School of Informatics, Xiamen University","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"Key Laboratory of Digital Protection and Intelligent Processing of Intangible Cultural Heritage of Fujian and Taiwan (Xiamen University), Ministry of Culture and Tourism","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I191208505"],"apc_list":null,"apc_paid":null,"fwci":1.8176,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.83819988,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"3860","last_page":"3864"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11398","display_name":"Hand Gesture Recognition Systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11398","display_name":"Hand Gesture Recognition Systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9890999794006348,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11285","display_name":"Hearing Impairment and Communication","score":0.9850999712944031,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7385112643241882},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.6967336535453796},{"id":"https://openalex.org/keywords/sign-language","display_name":"Sign language","score":0.6080343723297119},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.5981253981590271},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.5979495048522949},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5499472618103027},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5170103311538696},{"id":"https://openalex.org/keywords/gloss","display_name":"Gloss (optics)","score":0.5059255957603455},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.42481809854507446},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.3251398205757141},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.22638851404190063}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7385112643241882},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.6967336535453796},{"id":"https://openalex.org/C522192633","wikidata":"https://www.wikidata.org/wiki/Q34228","display_name":"Sign language","level":2,"score":0.6080343723297119},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.5981253981590271},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.5979495048522949},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5499472618103027},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5170103311538696},{"id":"https://openalex.org/C143025027","wikidata":"https://www.wikidata.org/wiki/Q900581","display_name":"Gloss (optics)","level":3,"score":0.5059255957603455},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.42481809854507446},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3251398205757141},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.22638851404190063},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C2781448156","wikidata":"https://www.wikidata.org/wiki/Q1570182","display_name":"Coating","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp48485.2024.10446966","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10446966","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.6000000238418579}],"awards":[{"id":"https://openalex.org/G4878625642","display_name":null,"funder_award_id":"62076211","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1965555277","https://openalex.org/W2101105183","https://openalex.org/W2154652894","https://openalex.org/W2799020610","https://openalex.org/W3034765865","https://openalex.org/W3114337930","https://openalex.org/W3133226919","https://openalex.org/W3173290664","https://openalex.org/W3175285983","https://openalex.org/W3175963743","https://openalex.org/W3176455679","https://openalex.org/W3203359574","https://openalex.org/W4312396348","https://openalex.org/W4313021454","https://openalex.org/W4367047590","https://openalex.org/W4372260090","https://openalex.org/W4385767762","https://openalex.org/W4393160661","https://openalex.org/W6631190155","https://openalex.org/W6682631176","https://openalex.org/W6790978476","https://openalex.org/W6796187045","https://openalex.org/W6846322403","https://openalex.org/W6849177959","https://openalex.org/W6852329775"],"related_works":["https://openalex.org/W73545470","https://openalex.org/W4224266612","https://openalex.org/W2383394264","https://openalex.org/W4320153225","https://openalex.org/W4293261942","https://openalex.org/W3125968744","https://openalex.org/W203959209","https://openalex.org/W2110287964","https://openalex.org/W2167701463","https://openalex.org/W4307407935"],"abstract_inverted_index":{"Sign":[0,48],"Language":[1,49],"Translation":[2,50],"(SLT)":[3],"aims":[4],"to":[5,51,72,99,119],"convert":[6],"sign":[7,21,76],"language":[8,22,77],"videos":[9],"into":[10],"corresponding":[11],"spoken":[12],"text":[13,25],"sequences.":[14],"However,":[15],"the":[16,27,33,53,57,89,101,109,140,145],"inherent":[17],"modality":[18,54,146],"gap":[19,55],"between":[20,36],"video":[23],"and":[24,68,74,78,115,129,142],"hinders":[26],"development":[28],"of":[29,59,111],"SLT.":[30],"Motivated":[31],"by":[32],"linguistic":[34],"consistency":[35,110],"gloss1and":[37],"text,":[38],"we":[39],"propose":[40],"EMF-SLT,":[41],"an":[42],"Explicit":[43],"Multi-modal":[44],"Fusion":[45],"method":[46,135],"for":[47,88],"mitigate":[52],"with":[56],"help":[58],"gloss.":[60],"Specifically,":[61],"EMF-SLT":[62],"first":[63],"leverages":[64],"a":[65,69,92],"vector":[66],"quantizer":[67],"fusion":[70],"module":[71],"align":[73],"fuse":[75],"gloss":[79],"features,":[80],"respectively,":[81],"resulting":[82],"in":[83],"more":[84],"informative":[85],"multi-modal":[86],"features":[87],"decoder.":[90],"Then,":[91],"multi-task":[93],"mutual":[94],"learning":[95],"framework":[96],"is":[97],"introduced":[98],"regularize":[100],"output":[102],"predictions":[103],"from":[104,121],"different":[105,117],"modalities,":[106],"which":[107],"ensures":[108],"outputs":[112],"across":[113],"modalities":[114,118],"encourages":[116],"learn":[120],"each":[122],"other.":[123],"Experiments":[124],"on":[125],"two":[126],"SLT":[127],"benchmarks":[128],"further":[130],"analyses":[131],"show":[132],"that":[133],"our":[134],"achieves":[136],"significant":[137],"improvements":[138],"over":[139],"baselines":[141],"effectively":[143],"alleviates":[144],"gap.":[147]},"counts_by_year":[{"year":2025,"cited_by_count":5}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}