{"id":"https://openalex.org/W4376481237","doi":"https://doi.org/10.1109/tmm.2023.3275873","title":"VatLM: Visual-Audio-Text Pre-Training With Unified Masked Prediction for Speech Representation Learning","display_name":"VatLM: Visual-Audio-Text Pre-Training With Unified Masked Prediction for Speech Representation Learning","publication_year":2023,"publication_date":"2023-05-12","ids":{"openalex":"https://openalex.org/W4376481237","doi":"https://doi.org/10.1109/tmm.2023.3275873"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2023.3275873","doi":"https://doi.org/10.1109/tmm.2023.3275873","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3275873","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-a","cwts-core","dongbi-a","jufo-3","norway-1","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2211.11275","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045966396","display_name":"Qiushi Zhu","orcid":"https://orcid.org/0000-0002-1196-7781","observed_orcids":["https://orcid.org/0000-0002-1196-7781"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qiushi Zhu","raw_affiliation_strings":["National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China, Hefei, China","National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China (USTC), Hefei, China"],"raw_orcid":"https://orcid.org/0000-0002-1196-7781","affiliations":[{"raw_affiliation_string":"National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]},{"raw_affiliation_string":"National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China (USTC), Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106557565","display_name":"Long Zhou","orcid":"https://orcid.org/0009-0006-1919-4943","observed_orcids":["https://orcid.org/0009-0006-1919-4943"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Long Zhou","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0006-1919-4943","affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101577318","display_name":"Ziqiang Zhang","orcid":"https://orcid.org/0000-0003-0110-1543","observed_orcids":["https://orcid.org/0000-0003-0110-1543"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziqiang Zhang","raw_affiliation_strings":["National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China, Hefei, China","National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China (USTC), Hefei, China"],"raw_orcid":"https://orcid.org/0000-0003-0110-1543","affiliations":[{"raw_affiliation_string":"National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]},{"raw_affiliation_string":"National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China (USTC), Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101635405","display_name":"Shujie Liu","orcid":"https://orcid.org/0009-0008-2599-6752","observed_orcids":["https://orcid.org/0009-0008-2599-6752"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shujie Liu","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045811313","display_name":"Binxing Jiao","orcid":"https://orcid.org/0000-0003-4710-0095","observed_orcids":["https://orcid.org/0000-0003-4710-0095"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Binxing Jiao","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0000-0003-4710-0095","affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100436848","display_name":"Jie Zhang","orcid":"https://orcid.org/0000-0003-1124-0854","observed_orcids":["https://orcid.org/0000-0003-1124-0854"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jie Zhang","raw_affiliation_strings":["National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China, Hefei, China","National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China (USTC), Hefei, China"],"raw_orcid":"https://orcid.org/0000-0003-1124-0854","affiliations":[{"raw_affiliation_string":"National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]},{"raw_affiliation_string":"National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China (USTC), Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057227915","display_name":"Li-Rong Dai","orcid":"https://orcid.org/0000-0002-0859-2827","observed_orcids":["https://orcid.org/0000-0002-0859-2827"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lirong Dai","raw_affiliation_strings":["National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China, Hefei, China","National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China (USTC), Hefei, China"],"raw_orcid":"https://orcid.org/0000-0002-0859-2827","affiliations":[{"raw_affiliation_string":"National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]},{"raw_affiliation_string":"National Engineering Research Center of Speech and Language Information Processing (NERC-SLIP), University of Science and Technology of China (USTC), Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060116992","display_name":"Daxin Jiang","orcid":"https://orcid.org/0000-0002-6657-5806","observed_orcids":["https://orcid.org/0000-0002-6657-5806"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daxin Jiang","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100365053","display_name":"Jinyu Li","orcid":"https://orcid.org/0000-0002-1089-9748","observed_orcids":["https://orcid.org/0000-0002-1089-9748"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jinyu Li","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-1089-9748","affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014662947","display_name":"Furu Wei","orcid":"https://orcid.org/0000-0002-7810-5852","observed_orcids":["https://orcid.org/0000-0002-7810-5852"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Furu Wei","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2345,"currency":"USD","value_usd":2345},"apc_paid":null,"fwci":3.4294,"has_fulltext":false,"cited_by_count":38,"citation_normalized_percentile":{"value":0.94222945,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"26","issue":null,"first_page":"1055","last_page":"1064"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.49000000953674316,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.49000000953674316,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.3910999894142151,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.08959999680519104,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multimodal-pretraining","display_name":"multimodal pre-training","score":0.7310000061988831},{"id":"https://openalex.org/keywords/speech-representation-learning","display_name":"speech representation learning","score":0.8659999966621399},{"id":"https://openalex.org/keywords/audio-visual-speech-recognition","display_name":"audio-visual speech recognition","score":0.8320000171661377},{"id":"https://openalex.org/keywords/visual-speech-recognition","display_name":"visual speech recognition","score":0.996999979019165},{"id":"https://openalex.org/keywords/masked-prediction","display_name":"masked prediction","score":0.625},{"id":"https://openalex.org/keywords/cross-modal-representation-learning","display_name":"cross-modal representation learning","score":0.8140000104904175}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8799227476119995},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6886312365531921},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5621363520622253},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5520034432411194},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5341383814811707},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5104386210441589},{"id":"https://openalex.org/C3017588708","wikidata":"https://www.wikidata.org/wiki/Q758901","display_name":"Audio visual","level":2,"score":0.4856904447078705},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4578576982021332},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.2731668949127197},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tmm.2023.3275873","doi":"https://doi.org/10.1109/tmm.2023.3275873","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3275873","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-a","cwts-core","dongbi-a","jufo-3","norway-1","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2211.11275","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2211.11275","pdf_url":"https://arxiv.org/pdf/2211.11275","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2211.11275","doi":"https://doi.org/10.48550/arxiv.2211.11275","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2211.11275","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2211.11275","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2211.11275","pdf_url":"https://arxiv.org/pdf/2211.11275","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"id":"https://metadata.un.org/sdg/16","score":0.46000000834465027,"display_name":"Peace, justice, and strong institutions"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G4358731121","display_name":null,"funder_award_id":"[2022]-012","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G634576251","display_name":null,"funder_award_id":"62101523","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8603829020","display_name":null,"funder_award_id":"XDC08010200","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320333993","display_name":"Microsoft Research Asia","ror":"https://ror.org/0300m5276"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":67,"referenced_works":["https://openalex.org/W1917432393","https://openalex.org/W2014621385","https://openalex.org/W2032686202","https://openalex.org/W2115252128","https://openalex.org/W2121486117","https://openalex.org/W2134867751","https://openalex.org/W2219249508","https://openalex.org/W2787944098","https://openalex.org/W2799473636","https://openalex.org/W2808631503","https://openalex.org/W2883383043","https://openalex.org/W2885185669","https://openalex.org/W2890952074","https://openalex.org/W2891205112","https://openalex.org/W2933138175","https://openalex.org/W2939710050","https://openalex.org/W2946200149","https://openalex.org/W2972943112","https://openalex.org/W2981501041","https://openalex.org/W2996970093","https://openalex.org/W3003133812","https://openalex.org/W3006974783","https://openalex.org/W3007589762","https://openalex.org/W3015383493","https://openalex.org/W3015830103","https://openalex.org/W3016010032","https://openalex.org/W3035042697","https://openalex.org/W3041561163","https://openalex.org/W3148001440","https://openalex.org/W3154596443","https://openalex.org/W3154807520","https://openalex.org/W3162293946","https://openalex.org/W3167917117","https://openalex.org/W3169320628","https://openalex.org/W3173978205","https://openalex.org/W3197567540","https://openalex.org/W3197580070","https://openalex.org/W3198694222","https://openalex.org/W3205193540","https://openalex.org/W3205644108","https://openalex.org/W3205715971","https://openalex.org/W3207222250","https://openalex.org/W3209371554","https://openalex.org/W3209984917","https://openalex.org/W4221153068","https://openalex.org/W4221153521","https://openalex.org/W4221155340","https://openalex.org/W4223622550","https://openalex.org/W4224319127","https://openalex.org/W4226002507","https://openalex.org/W4226033575","https://openalex.org/W4226120743","https://openalex.org/W4281492411","https://openalex.org/W4297808394","https://openalex.org/W4297841641","https://openalex.org/W4307286264","https://openalex.org/W4308236834","https://openalex.org/W4312638101","https://openalex.org/W4382202943","https://openalex.org/W4385245566","https://openalex.org/W4385573012","https://openalex.org/W4386071687","https://openalex.org/W4392979802","https://openalex.org/W6688816777","https://openalex.org/W6754420807","https://openalex.org/W6780218876","https://openalex.org/W6810007534"],"related_works":["https://openalex.org/W2271369634","https://openalex.org/W3147472394","https://openalex.org/W2047100085","https://openalex.org/W2350550760","https://openalex.org/W578794879","https://openalex.org/W2625296515","https://openalex.org/W230091440","https://openalex.org/W3137890128","https://openalex.org/W4394050964","https://openalex.org/W2551249631"],"abstract_inverted_index":{"Although":[0],"speech":[1,19,54,135,140],"is":[2],"a":[3,16,30,66,76,112],"simple":[4,88],"and":[5,38,49,85,95,138,159,181],"effective":[6],"way":[7],"for":[8],"humans":[9],"to":[10,28,33,52,80,91,100],"communicate":[11],"with":[12,111],"the":[13,82,127,147,154,170,179],"outside":[14],"world,":[15],"more":[17],"realistic":[18],"interaction":[20],"contains":[21],"multimodal":[22],"information,":[23],"e.g.,":[24],"vision,":[25],"text.":[26],"How":[27],"design":[29],"unified":[31,67,77,117,123],"framework":[32],"integrate":[34,101],"different":[35,40,167],"modal":[36],"information":[37,84],"leverage":[39],"resources":[41],"(e.g.,":[42],"visual-audio":[43],"pairs,":[44,46],"audio-text":[45],"unlabeled":[47,50],"speech,":[48,94],"text)":[51],"facilitate":[53,174],"representation":[55,69],"learning":[56,70],"was":[57],"not":[58],"well":[59],"explored.":[60],"In":[61,98],"this":[62],"paper,":[63],"we":[64,177],"propose":[65],"cross-modal":[68],"frameworkVatLM(Visual-Audio-Text":[71],"Language":[72],"Model).":[73],"The":[74],"proposedVatLMemploys":[75],"backbone":[78],"network":[79],"model":[81],"modality-independent":[83],"utilizes":[86],"three":[87,103],"modality-dependent":[89],"modules":[90],"preprocess":[92],"visual,":[93],"text":[96],"inputs.":[97],"order":[99],"these":[102],"modalities":[104,168],"into":[105,169],"one":[106],"shared":[107],"semantic":[108],"space,VatLMis":[109],"optimized":[110],"masked":[113],"prediction":[114],"task":[115],"of":[116,165],"tokens,":[118],"given":[119],"by":[120],"our":[121],"proposed":[122],"tokenizer.":[124],"We":[125],"evaluate":[126],"pre-trainedVatLMon":[128],"audio-visual":[129,134,155],"related":[130],"downstream":[131],"tasks,":[132],"including":[133],"recognition":[136,141],"(AVSR),":[137],"visual":[139],"(VSR)":[142],"tasks.":[143],"Results":[144],"show":[145],"that":[146],"proposedVatLMoutperforms":[148],"previous":[149],"state-of-the-art":[150],"models,":[151],"such":[152],"as":[153],"pre-trained":[156,182],"AV-HuBERT":[157],"model,":[158],"analysis":[160],"also":[161],"demonstrates":[162],"thatVatLMis":[163],"capable":[164],"aligning":[166],"same":[171],"space.":[172],"To":[173],"future":[175],"research,":[176],"release":[178],"code":[180],"models":[183],"athttps://aka.ms/vatlm.":[184]},"counts_by_year":[{"year":2026,"cited_by_count":11},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":4}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2023-05-14T00:00:00"}