{"id":"https://openalex.org/W4403780768","doi":"https://doi.org/10.1145/3664647.3681221","title":"Efficient Training for Multilingual Visual Speech Recognition: Pre-training with Discretized Visual Speech Representation","display_name":"Efficient Training for Multilingual Visual Speech Recognition: Pre-training with Discretized Visual Speech Representation","publication_year":2024,"publication_date":"2024-10-26","ids":{"openalex":"https://openalex.org/W4403780768","doi":"https://doi.org/10.1145/3664647.3681221"},"language":"en","primary_location":{"id":"doi:10.1145/3664647.3681221","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3664647.3681221","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3664647.3681221","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100343629","display_name":"Minsu Kim","orcid":"https://orcid.org/0000-0002-6514-0018"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Minsu Kim","raw_affiliation_strings":["Korea Advanced Institute of Science & Technology, Daejeon, Republic of Korea"],"raw_orcid":"https://orcid.org/0000-0002-6514-0018","affiliations":[{"raw_affiliation_string":"Korea Advanced Institute of Science & Technology, Daejeon, Republic of Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089479205","display_name":"Jeong Hun Yeo","orcid":"https://orcid.org/0009-0002-8135-6625"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jeonghun Yeo","raw_affiliation_strings":["Korea Advanced Institute of Science & Technology, Daejeon, Republic of Korea"],"raw_orcid":"https://orcid.org/0009-0002-8135-6625","affiliations":[{"raw_affiliation_string":"Korea Advanced Institute of Science & Technology, Daejeon, Republic of Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100376688","display_name":"Se Jin Park","orcid":"https://orcid.org/0000-0001-8467-3576"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Se Jin Park","raw_affiliation_strings":["Korea Advanced Institute of Science & Technology, Daejeon, Republic of Korea"],"raw_orcid":"https://orcid.org/0000-0001-8467-3576","affiliations":[{"raw_affiliation_string":"Korea Advanced Institute of Science & Technology, Daejeon, Republic of Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036329402","display_name":"Hyeongseop Rha","orcid":null},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Hyeongseop Rha","raw_affiliation_strings":["Korea Advanced Institute of Science & Technology, Daejeon, Republic of Korea"],"raw_orcid":"https://orcid.org/0009-0004-9301-2760","affiliations":[{"raw_affiliation_string":"Korea Advanced Institute of Science & Technology, Daejeon, Republic of Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5038798134","display_name":"Yong Man Ro","orcid":"https://orcid.org/0000-0001-5306-6853"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Yong Man Ro","raw_affiliation_strings":["Korea Advanced Institute of Science & Technology, Daejeon, Republic of Korea"],"raw_orcid":"https://orcid.org/0000-0001-5306-6853","affiliations":[{"raw_affiliation_string":"Korea Advanced Institute of Science & Technology, Daejeon, Republic of Korea","institution_ids":["https://openalex.org/I157485424"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.9164,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.7985524,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1311","last_page":"1320"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.78322434425354},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.7769516706466675},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.7120920419692993},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.5102944374084473},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.46406087279319763},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.45979663729667664},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.44648104906082153},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.41115012764930725}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.78322434425354},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.7769516706466675},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.7120920419692993},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5102944374084473},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46406087279319763},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.45979663729667664},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.44648104906082153},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.41115012764930725},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3664647.3681221","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3664647.3681221","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3664647.3681221","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3664647.3681221","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6399999856948853}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W2042718506","https://openalex.org/W2194775991","https://openalex.org/W2404704342","https://openalex.org/W2766219058","https://openalex.org/W2808631503","https://openalex.org/W2963192365","https://openalex.org/W2963431393","https://openalex.org/W2963528589","https://openalex.org/W2963654155","https://openalex.org/W2963785710","https://openalex.org/W2964309797","https://openalex.org/W2996970093","https://openalex.org/W3008549139","https://openalex.org/W3015830103","https://openalex.org/W3024869864","https://openalex.org/W3034552680","https://openalex.org/W3095410713","https://openalex.org/W3129009457","https://openalex.org/W3140429000","https://openalex.org/W3162293946","https://openalex.org/W3162707322","https://openalex.org/W3163793923","https://openalex.org/W3167917117","https://openalex.org/W3205193540","https://openalex.org/W3209059054","https://openalex.org/W3209984917","https://openalex.org/W4224319127","https://openalex.org/W4225299282","https://openalex.org/W4225746985","https://openalex.org/W4287854499","https://openalex.org/W4289665794","https://openalex.org/W4296070387","https://openalex.org/W4307286264","https://openalex.org/W4307680525","https://openalex.org/W4312638101","https://openalex.org/W4372346152","https://openalex.org/W4375868953","https://openalex.org/W4376481237","https://openalex.org/W4385822683","https://openalex.org/W4385823403","https://openalex.org/W4386071467","https://openalex.org/W4390691978","https://openalex.org/W4390871839","https://openalex.org/W4390874021","https://openalex.org/W4394671563"],"related_works":["https://openalex.org/W230091440","https://openalex.org/W2068608913","https://openalex.org/W2233261550","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2997094352","https://openalex.org/W3124914020","https://openalex.org/W3216976533","https://openalex.org/W4394050964","https://openalex.org/W2551249631"],"abstract_inverted_index":{"This":[0],"paper":[1],"explores":[2],"sentence-level":[3],"multilingual":[4,22,85,169],"Visual":[5],"Speech":[6],"Recognition":[7],"(VSR)":[8],"that":[9,47],"can":[10,102],"recognize":[11],"different":[12],"languages":[13],"with":[14,39,145,182],"a":[15,33,76,183],"single":[16,184],"trained":[17,185],"model.":[18,186],"As":[19,87],"the":[20,48,62,67,89,105,110,115,124,132,139,142,158,177],"massive":[21],"modeling":[23],"of":[24,69,123,141],"visual":[25,40,49,63,92,153],"data":[26,117],"requires":[27],"huge":[28],"computational":[29],"costs,":[30],"we":[31,45,72,101,134],"propose":[32,73],"novel":[34],"efficient":[35],"training":[36,106],"strategy,":[37],"processing":[38],"speech":[41,50,64,93,147,154],"units.":[42,155],"Through":[43],"analysis,":[44],"confirm":[46],"units":[51,65,148],"mainly":[52],"contain":[53],"viseme":[54],"information":[55],"while":[56],"suppressing":[57],"non-linguistic":[58],"information.":[59],"By":[60],"using":[61],"as":[66],"inputs":[68,90,140],"our":[70],"system,":[71],"to":[74,79,109,121,130,152,176],"pre-train":[75],"VSR":[77,112,170,180],"model":[78,159],"predict":[80],"corresponding":[81],"text":[82],"outputs":[83,96],"on":[84,162],"data.":[86],"both":[88],"(i.e.,":[91,97],"units)":[94],"and":[95,149],"text)":[98],"are":[99],"discrete,":[100],"greatly":[103],"improve":[104],"efficiency":[107],"compared":[108],"standard":[111],"training.":[113],"Specifically,":[114],"input":[116],"size":[118],"is":[119,160],"reduced":[120],"0.016%":[122],"original":[125],"video":[126],"inputs.":[127],"In":[128],"addition,":[129],"stabilize":[131],"training,":[133],"apply":[135],"curriculum":[136],"learning":[137],"where":[138],"system":[143],"begin":[144],"audio-visual":[146],"gradually":[150],"transition":[151],"After":[156],"pre-training,":[157],"finetuned":[161],"continuous":[163],"features.":[164],"We":[165],"set":[166],"new":[167],"state-of-the-art":[168],"performances":[171,175],"by":[172],"achieving":[173],"comparable":[174],"previous":[178],"language-specific":[179],"models,":[181]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}