{"id":"https://openalex.org/W4392932002","doi":"https://doi.org/10.1109/icassp48485.2024.10446720","title":"Visual Speech Recognition for Languages with Limited Labeled Data Using Automatic Labels from Whisper","display_name":"Visual Speech Recognition for Languages with Limited Labeled Data Using Automatic Labels from Whisper","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4392932002","doi":"https://doi.org/10.1109/icassp48485.2024.10446720"},"language":"en","primary_location":{"id":"doi:10.1109/icassp48485.2024.10446720","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10446720","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089479205","display_name":"Jeong Hun Yeo","orcid":"https://orcid.org/0009-0002-8135-6625"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jeong Hun Yeo","raw_affiliation_strings":["KAIST,Integrated Vision and Language Lab,South Korea","Integrated Vision and Language Lab, KAIST, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAIST,Integrated Vision and Language Lab,South Korea","institution_ids":["https://openalex.org/I157485424"]},{"raw_affiliation_string":"Integrated Vision and Language Lab, KAIST, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100343629","display_name":"Minsu Kim","orcid":"https://orcid.org/0000-0002-6514-0018"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Minsu Kim","raw_affiliation_strings":["KAIST,Integrated Vision and Language Lab,South Korea","Integrated Vision and Language Lab, KAIST, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAIST,Integrated Vision and Language Lab,South Korea","institution_ids":["https://openalex.org/I157485424"]},{"raw_affiliation_string":"Integrated Vision and Language Lab, KAIST, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001291873","display_name":"Shinji Watanabe","orcid":"https://orcid.org/0000-0002-5970-8631"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shinji Watanabe","raw_affiliation_strings":["Carnegie Mellon University,Language Technologies Institute,USA","Language Technologies Institute, Carnegie Mellon University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,Language Technologies Institute,USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Language Technologies Institute, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5038798134","display_name":"Yong Man Ro","orcid":"https://orcid.org/0000-0001-5306-6853"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Yong Man Ro","raw_affiliation_strings":["KAIST,Integrated Vision and Language Lab,South Korea","Integrated Vision and Language Lab, KAIST, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAIST,Integrated Vision and Language Lab,South Korea","institution_ids":["https://openalex.org/I157485424"]},{"raw_affiliation_string":"Integrated Vision and Language Lab, KAIST, South Korea","institution_ids":["https://openalex.org/I157485424"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"10471","last_page":"10475"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8690510392189026},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5871113538742065},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5809690356254578},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4789600670337677},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.42985936999320984},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.41920024156570435},{"id":"https://openalex.org/keywords/labeled-data","display_name":"Labeled data","score":0.4140496850013733},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.07610070705413818}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8690510392189026},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5871113538742065},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5809690356254578},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4789600670337677},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.42985936999320984},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.41920024156570435},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.4140496850013733},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.07610070705413818}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp48485.2024.10446720","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10446720","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320320671","display_name":"National Research Foundation","ror":"https://ror.org/05s0g1g46"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W1922655562","https://openalex.org/W2091409162","https://openalex.org/W2194775991","https://openalex.org/W2551572271","https://openalex.org/W2594690981","https://openalex.org/W2766219058","https://openalex.org/W2808631503","https://openalex.org/W2890952074","https://openalex.org/W2891205112","https://openalex.org/W2963250244","https://openalex.org/W2963654155","https://openalex.org/W2981501041","https://openalex.org/W2996970093","https://openalex.org/W3015830103","https://openalex.org/W3034552680","https://openalex.org/W3097777922","https://openalex.org/W3100921325","https://openalex.org/W3162293946","https://openalex.org/W3163793923","https://openalex.org/W3197771105","https://openalex.org/W3211278025","https://openalex.org/W4224319127","https://openalex.org/W4289665794","https://openalex.org/W4298112588","https://openalex.org/W4307286264","https://openalex.org/W4312920990","https://openalex.org/W4372260478","https://openalex.org/W4372266842","https://openalex.org/W4372346152","https://openalex.org/W4375868850","https://openalex.org/W4385245566","https://openalex.org/W4385822683","https://openalex.org/W4385823403","https://openalex.org/W4388017359","https://openalex.org/W4390691978","https://openalex.org/W4390874021","https://openalex.org/W4392908934","https://openalex.org/W6631190155","https://openalex.org/W6640090968","https://openalex.org/W6687566353","https://openalex.org/W6732872814","https://openalex.org/W6754420807","https://openalex.org/W6757817989","https://openalex.org/W6810168380","https://openalex.org/W6847363464"],"related_works":["https://openalex.org/W2130553454","https://openalex.org/W3022007134","https://openalex.org/W4317548404","https://openalex.org/W2087783760","https://openalex.org/W1509924131","https://openalex.org/W3104108945","https://openalex.org/W3163689946","https://openalex.org/W4287890939","https://openalex.org/W2033364610","https://openalex.org/W2797776314"],"abstract_inverted_index":{"This":[0],"paper":[1],"proposes":[2],"a":[3,19,71],"powerful":[4],"Visual":[5],"Speech":[6],"Recognition":[7],"(VSR)":[8],"method":[9],"for":[10,14,36,59,156],"multiple":[11],"languages,":[12,46,161,180],"especially":[13],"low-resource":[15],"languages":[16,62,92],"that":[17,29,120,128],"have":[18],"limited":[20],"number":[21],"of":[22,55,89,107,129,154],"labeled":[23],"data.":[24],"Different":[25],"from":[26,44,96],"previous":[27,184],"methods":[28],"tried":[30],"to":[31,86,127],"improve":[32],"the":[33,37,53,60,90,97,105,115,138,168,183],"VSR":[34,108,125,159],"performance":[35,126,175],"target":[38],"language":[39,78],"by":[40],"using":[41],"knowledge":[42],"learned":[43],"other":[45],"we":[47,50,69,118,121,142,171],"explore":[48],"whether":[49],"can":[51,75,122],"increase":[52],"amount":[54],"training":[56],"data":[57,88,101,155],"itself":[58],"different":[61],"without":[63,133],"human":[64,135],"intervention.":[65],"To":[66],"this":[67],"end,":[68],"employ":[70],"Whisper":[72],"model":[73],"which":[74],"conduct":[76],"both":[77],"identification":[79],"and":[80,93,114,149,165],"audio-based":[81],"speech":[82],"recognition.":[83],"It":[84],"serves":[85],"filter":[87],"desired":[91],"transcribe":[94],"labels":[95,113,131,188],"unannotated,":[98],"multilingual":[99,146],"audio-visual":[100],"pool.":[102],"By":[103],"comparing":[104],"performances":[106],"models":[109],"trained":[110],"on":[111,176],"automatic":[112,169,187],"human-annotated":[116,130],"labels,":[117,170],"show":[119],"achieve":[123,172],"similar":[124],"even":[132],"utilizing":[134],"annotations.":[136],"Through":[137],"automated":[139],"labeling":[140],"process,":[141],"label":[143],"large-scale":[144],"unlabeled":[145],"databases,":[147],"VoxCeleb2":[148],"AVSpeech,":[150],"producing":[151],"1,002":[152],"hours":[153],"four":[157,179],"low":[158],"resource":[160],"French,":[162],"Italian,":[163],"Spanish,":[164],"Portuguese.":[166],"With":[167],"new":[173],"state-of-the-art":[174],"mTEDx":[177],"in":[178],"significantly":[181],"surpassing":[182],"methods.":[185],"The":[186],"are":[189],"available":[190],"online:":[191],"bit.ly/3Lajr6w":[192]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}