{"id":"https://openalex.org/W4408354236","doi":"https://doi.org/10.1109/icassp49660.2025.10889251","title":"Large Language Models are Strong Audio-Visual Speech Recognition Learners","display_name":"Large Language Models are Strong Audio-Visual Speech Recognition Learners","publication_year":2025,"publication_date":"2025-03-12","ids":{"openalex":"https://openalex.org/W4408354236","doi":"https://doi.org/10.1109/icassp49660.2025.10889251"},"language":"en","primary_location":{"id":"doi:10.1109/icassp49660.2025.10889251","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp49660.2025.10889251","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2025 - 2025 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043797502","display_name":"Umberto Cappellazzo","orcid":"https://orcid.org/0009-0006-3443-5143"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Umberto Cappellazzo","raw_affiliation_strings":["Imperial College London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Imperial College London","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100343643","display_name":"Minsu Kim","orcid":"https://orcid.org/0000-0003-4882-0959"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Minsu Kim","raw_affiliation_strings":["Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta AI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081576515","display_name":"Honglie Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Honglie Chen","raw_affiliation_strings":["Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta AI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101728918","display_name":"Pingchuan Ma","orcid":"https://orcid.org/0000-0001-7680-2817"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pingchuan Ma","raw_affiliation_strings":["Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta AI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009475700","display_name":"Stavros Petridis","orcid":"https://orcid.org/0000-0001-7478-9479"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Stavros Petridis","raw_affiliation_strings":["Imperial College London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Imperial College London","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041102462","display_name":"Daniele Falavigna","orcid":"https://orcid.org/0000-0002-4844-5071"},"institutions":[{"id":"https://openalex.org/I2277624104","display_name":"Fondazione Bruno Kessler","ror":"https://ror.org/01j33xk10","country_code":"IT","type":"facility","lineage":["https://openalex.org/I2277624104"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Daniele Falavigna","raw_affiliation_strings":["Fondazione Bruno Kessler"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fondazione Bruno Kessler","institution_ids":["https://openalex.org/I2277624104"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066363315","display_name":"Alessio Brutti","orcid":"https://orcid.org/0000-0003-4146-3071"},"institutions":[{"id":"https://openalex.org/I2277624104","display_name":"Fondazione Bruno Kessler","ror":"https://ror.org/01j33xk10","country_code":"IT","type":"facility","lineage":["https://openalex.org/I2277624104"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Alessio Brutti","raw_affiliation_strings":["Fondazione Bruno Kessler"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fondazione Bruno Kessler","institution_ids":["https://openalex.org/I2277624104"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016033078","display_name":"Maja Panti\u0107","orcid":"https://orcid.org/0000-0002-3620-5986"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Maja Pantic","raw_affiliation_strings":["Imperial College London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Imperial College London","institution_ids":["https://openalex.org/I47508984"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":31.7873,"has_fulltext":false,"cited_by_count":19,"citation_normalized_percentile":{"value":0.99820965,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9135000109672546,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9135000109672546,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.900600016117096,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7924734950065613},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6787512898445129},{"id":"https://openalex.org/keywords/audio-visual","display_name":"Audio visual","score":0.6389109492301941},{"id":"https://openalex.org/keywords/audio-mining","display_name":"Audio mining","score":0.47132423520088196},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3914913833141327},{"id":"https://openalex.org/keywords/speech-processing","display_name":"Speech processing","score":0.338936448097229},{"id":"https://openalex.org/keywords/acoustic-model","display_name":"Acoustic model","score":0.32684504985809326},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.281119704246521}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7924734950065613},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6787512898445129},{"id":"https://openalex.org/C3017588708","wikidata":"https://www.wikidata.org/wiki/Q758901","display_name":"Audio visual","level":2,"score":0.6389109492301941},{"id":"https://openalex.org/C157968479","wikidata":"https://www.wikidata.org/wiki/Q3079876","display_name":"Audio mining","level":4,"score":0.47132423520088196},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3914913833141327},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.338936448097229},{"id":"https://openalex.org/C155635449","wikidata":"https://www.wikidata.org/wiki/Q4674699","display_name":"Acoustic model","level":3,"score":0.32684504985809326},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.281119704246521}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp49660.2025.10889251","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp49660.2025.10889251","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2025 - 2025 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6600000262260437}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":48,"referenced_works":["https://openalex.org/W1974387177","https://openalex.org/W2808631503","https://openalex.org/W2890952074","https://openalex.org/W3162293946","https://openalex.org/W3209059054","https://openalex.org/W3209984917","https://openalex.org/W4297841641","https://openalex.org/W4297841719","https://openalex.org/W4307286264","https://openalex.org/W4372346152","https://openalex.org/W4385807453","https://openalex.org/W4391021811","https://openalex.org/W4392903321","https://openalex.org/W4392903956","https://openalex.org/W4392904484","https://openalex.org/W4392908934","https://openalex.org/W4392931626","https://openalex.org/W4392932002","https://openalex.org/W4399728346","https://openalex.org/W4402112360","https://openalex.org/W4402727764","https://openalex.org/W4402727885","https://openalex.org/W4402753980","https://openalex.org/W4403002096","https://openalex.org/W4404002404","https://openalex.org/W4404784428","https://openalex.org/W4404792841","https://openalex.org/W4406461808","https://openalex.org/W4407451021","https://openalex.org/W6754420807","https://openalex.org/W6810168380","https://openalex.org/W6810334672","https://openalex.org/W6839936984","https://openalex.org/W6847363464","https://openalex.org/W6847652939","https://openalex.org/W6849177959","https://openalex.org/W6851592950","https://openalex.org/W6852818750","https://openalex.org/W6853094705","https://openalex.org/W6854866820","https://openalex.org/W6860549370","https://openalex.org/W6860679478","https://openalex.org/W6860809563","https://openalex.org/W6861152725","https://openalex.org/W6861626678","https://openalex.org/W6861859833","https://openalex.org/W6869892155","https://openalex.org/W6870122244"],"related_works":["https://openalex.org/W22517275","https://openalex.org/W4383605217","https://openalex.org/W4281389974","https://openalex.org/W2131711534","https://openalex.org/W1911859126","https://openalex.org/W2537070497","https://openalex.org/W1560013842","https://openalex.org/W1536752851","https://openalex.org/W2964829415","https://openalex.org/W642007152"],"abstract_inverted_index":{"Multimodal":[0],"large":[1],"language":[2],"models":[3],"(MLLMs)":[4],"have":[5,76],"recently":[6],"become":[7],"a":[8,89,118,134,184],"focal":[9],"point":[10],"of":[11,137,179,186,204,208,217],"research":[12],"due":[13],"to":[14,54,105,123],"their":[15],"formidable":[16],"multimodal":[17],"understanding":[18],"capabilities.":[19,97],"For":[20],"example,":[21],"in":[22,128],"the":[23,42,51,59,112,125,150,165,177,197,202,206,209,214,221],"audio":[24,43,48,101],"and":[25,50,64,102,144,153,170,181,188,212,220],"speech":[26,35,66,95],"domains,":[27],"an":[28,47,129],"LLM":[29,120,154],"can":[30],"be":[31],"equipped":[32],"with":[33,46,92,111,183],"(automatic)":[34],"recognition":[36,67,96],"(ASR)":[37],"abilities":[38],"by":[39,117],"just":[40],"concatenating":[41],"tokens,":[44,114],"computed":[45],"encoder,":[49],"text":[52,113],"tokens":[53,108],"achieve":[55,172],"state-of-the-art":[56,174],"results.":[57],"On":[58],"contrary,":[60],"tasks":[61,178],"like":[62],"visual":[63],"audio-visual":[65,94],"(VSR/AVSR),":[68],"which":[69],"also":[70],"exploit":[71],"noise-invariant":[72],"lip":[73],"movement":[74],"information,":[75],"received":[77],"little":[78],"or":[79],"no":[80],"attention.":[81],"To":[82,191],"bridge":[83],"this":[84],"gap,":[85],"we":[86,171,195],"propose":[87],"Llama-AVSR,":[88],"new":[90,173],"MLLM":[91],"strong":[93],"It":[98],"leverages":[99],"pre-trained":[100,119,210],"video":[103],"encoders":[104,152,211],"produce":[106],"modality-specific":[107,142],"which,":[109],"together":[110],"are":[115,147,155],"processed":[116],"(e.g.,":[121],"Llama3.1-8B)":[122],"yield":[124],"resulting":[126],"response":[127],"auto-regressive":[130],"fashion.":[131],"Llama-AVSR":[132],"requires":[133],"small":[135],"number":[136],"trainable":[138],"parameters":[139],"as":[140],"only":[141],"projectors":[143],"LoRA":[145,218],"modules":[146],"trained":[148],"whereas":[149],"multi-modal":[151],"kept":[156],"frozen.":[157],"We":[158],"evaluate":[159],"our":[160,193],"proposed":[161],"approach":[162],"on":[163],"LRS3,":[164],"largest":[166],"public":[167],"AVSR":[168,182],"benchmark,":[169],"results":[175],"for":[176],"ASR":[180],"WER":[185],"0.79%":[187],"0.77%,":[189],"respectively.":[190],"bolster":[192],"results,":[194],"investigate":[196],"key":[198],"factors":[199],"that":[200],"underpin":[201],"effectiveness":[203],"Llama-AVSR:":[205],"choice":[207],"LLM,":[213],"efficient":[215],"integration":[216],"modules,":[219],"optimal":[222],"performance-efficiency":[223],"trade-off":[224],"obtained":[225],"via":[226],"modality-aware":[227],"compression":[228],"rates.":[229]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":14}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}