{"id":"https://openalex.org/W3095070844","doi":"https://doi.org/10.21437/interspeech.2020-2359","title":"TMT: A Transformer-Based Modal Translator for Improving Multimodal Sequence Representations in Audio Visual Scene-Aware Dialog","display_name":"TMT: A Transformer-Based Modal Translator for Improving Multimodal Sequence Representations in Audio Visual Scene-Aware Dialog","publication_year":2020,"publication_date":"2020-10-25","ids":{"openalex":"https://openalex.org/W3095070844","doi":"https://doi.org/10.21437/interspeech.2020-2359","mag":"3095070844"},"language":"en","primary_location":{"id":"doi:10.21437/interspeech.2020-2359","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2020-2359","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2020","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050050390","display_name":"Wubo Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wubo Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110582135","display_name":"Dongwei Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dongwei Jiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108286207","display_name":"Wei Zou","orcid":"https://orcid.org/0000-0003-4215-5361"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei Zou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5081173423","display_name":"Xiangang Li","orcid":"https://orcid.org/0000-0002-7810-1077"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiangang Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.984000027179718,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dialog-box","display_name":"Dialog box","score":0.8111081719398499},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8024305105209351},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.7457112073898315},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5802944302558899},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5366036295890808},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.5188471078872681},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5011897087097168},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.4853162169456482},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.452843576669693},{"id":"https://openalex.org/keywords/multimodality","display_name":"Multimodality","score":0.41933491826057434},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.36213523149490356},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.340131938457489},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09260541200637817},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.06566187739372253}],"concepts":[{"id":"https://openalex.org/C173853756","wikidata":"https://www.wikidata.org/wiki/Q86915","display_name":"Dialog box","level":2,"score":0.8111081719398499},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8024305105209351},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.7457112073898315},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5802944302558899},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5366036295890808},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.5188471078872681},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5011897087097168},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.4853162169456482},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.452843576669693},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.41933491826057434},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36213523149490356},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.340131938457489},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09260541200637817},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.06566187739372253},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.21437/interspeech.2020-2359","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2020-2359","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2020","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.47999998927116394}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3176018525","https://openalex.org/W2903533908","https://openalex.org/W3026554633","https://openalex.org/W2903810591","https://openalex.org/W4289548192","https://openalex.org/W2888520903","https://openalex.org/W2963499882","https://openalex.org/W2903399267","https://openalex.org/W2949454572","https://openalex.org/W3098873988"],"abstract_inverted_index":{"Audio":[0],"Visual":[1],"Scene-aware":[2],"Dialog":[3,105],"(AVSD)":[4],"is":[5],"a":[6,14,76],"task":[7,26,123],"to":[8,54,69,88,140],"generate":[9],"responses":[10],"when":[11],"discussing":[12],"about":[13],"given":[15],"video.":[16],"The":[17],"previous":[18],"state-of-the-art":[19],"model":[20],"shows":[21],"superior":[22],"performance":[23],"for":[24,94],"this":[25],"using":[27],"Transformer-based":[28,50],"architecture.":[29],"However,":[30],"there":[31],"remain":[32],"some":[33],"limitations":[34],"in":[35,75,118],"learning":[36,155],"better":[37],"representation":[38],"of":[39,58,103],"modalities.":[40],"Inspired":[41],"by":[42,63],"Neural":[43],"Machine":[44],"Translation":[45],"(NMT),":[46],"we":[47,85],"propose":[48],"the":[49,56,59,65,70,95,100,104,112],"Modal":[51],"Translator":[52],"(TMT)":[53],"learn":[55],"representations":[57],"source":[60,66],"modal":[61,67,73],"sequence":[62,68,74],"translating":[64],"related":[71],"target":[72],"supervised":[77],"manner.":[78],"Based":[79],"on":[80,142],"Multimodal":[81],"Transformer":[82],"Networks":[83],"(MTN),":[84],"apply":[86],"TMT":[87],"video":[89],"and":[90,114,121,124],"dialog,":[91,150],"proposing":[92],"MTN-TMT":[93,110,131],"video-grounded":[96],"dialog":[97],"system.":[98],"On":[99],"AVSD":[101],"track":[102],"System":[106],"Technology":[107],"Challenge":[108],"7,":[109],"outperforms":[111],"MTN":[113],"other":[115],"submission":[116],"models":[117],"both":[119],"Video":[120],"Text":[122,125],"Only":[126],"task.":[127],"Compared":[128],"with":[129],"MTN,":[130],"improves":[132],"all":[133],"metrics,":[134],"especially,":[135],"achieving":[136],"relative":[137],"improvement":[138],"up":[139],"14.1%":[141],"CIDEr.":[143],"Index":[144],"Terms:":[145],"multimodal":[146],"learning,":[147],"audio-visual":[148],"scene-aware":[149],"neural":[151],"machine":[152],"translation,":[153],"multi-task":[154]},"counts_by_year":[{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}