{"id":"https://openalex.org/W4392904292","doi":"https://doi.org/10.1109/icassp48485.2024.10446888","title":"Towards Practical and Efficient Image-to-Speech Captioning with Vision-Language Pre-Training and Multi-Modal Tokens","display_name":"Towards Practical and Efficient Image-to-Speech Captioning with Vision-Language Pre-Training and Multi-Modal Tokens","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4392904292","doi":"https://doi.org/10.1109/icassp48485.2024.10446888"},"language":"en","primary_location":{"id":"doi:10.1109/icassp48485.2024.10446888","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10446888","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100343629","display_name":"Minsu Kim","orcid":"https://orcid.org/0000-0002-6514-0018"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Minsu Kim","raw_affiliation_strings":["KAIST,Integrated Vision and Language Lab,South Korea","Integrated Vision and Language Lab, KAIST, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAIST,Integrated Vision and Language Lab,South Korea","institution_ids":["https://openalex.org/I157485424"]},{"raw_affiliation_string":"Integrated Vision and Language Lab, KAIST, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101725649","display_name":"Jeongsoo Choi","orcid":"https://orcid.org/0009-0005-6817-604X"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jeongsoo Choi","raw_affiliation_strings":["KAIST,Integrated Vision and Language Lab,South Korea","Integrated Vision and Language Lab, KAIST, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAIST,Integrated Vision and Language Lab,South Korea","institution_ids":["https://openalex.org/I157485424"]},{"raw_affiliation_string":"Integrated Vision and Language Lab, KAIST, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010858961","display_name":"Soumi Maiti","orcid":"https://orcid.org/0000-0001-6940-0115"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Soumi Maiti","raw_affiliation_strings":["Carnegie Mellon University,Language Technologies Institute,USA","Language Technologies Institute, Carnegie Mellon University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,Language Technologies Institute,USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Language Technologies Institute, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089479205","display_name":"Jeong Hun Yeo","orcid":"https://orcid.org/0009-0002-8135-6625"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jeong Hun Yeo","raw_affiliation_strings":["KAIST,Integrated Vision and Language Lab,South Korea","Integrated Vision and Language Lab, KAIST, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAIST,Integrated Vision and Language Lab,South Korea","institution_ids":["https://openalex.org/I157485424"]},{"raw_affiliation_string":"Integrated Vision and Language Lab, KAIST, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001291873","display_name":"Shinji Watanabe","orcid":"https://orcid.org/0000-0002-5970-8631"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shinji Watanabe","raw_affiliation_strings":["Carnegie Mellon University,Language Technologies Institute,USA","Language Technologies Institute, Carnegie Mellon University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,Language Technologies Institute,USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Language Technologies Institute, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5038798134","display_name":"Yong Man Ro","orcid":"https://orcid.org/0000-0001-5306-6853"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Yong Man Ro","raw_affiliation_strings":["KAIST,Integrated Vision and Language Lab,South Korea","Integrated Vision and Language Lab, KAIST, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAIST,Integrated Vision and Language Lab,South Korea","institution_ids":["https://openalex.org/I157485424"]},{"raw_affiliation_string":"Integrated Vision and Language Lab, KAIST, South Korea","institution_ids":["https://openalex.org/I157485424"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"7970","last_page":"7974"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9921000003814697,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.8942902088165283},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8375104665756226},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.7317570447921753},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5476871132850647},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5340332984924316},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.4817497730255127},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.45513689517974854},{"id":"https://openalex.org/keywords/spoken-language","display_name":"Spoken language","score":0.44543343782424927},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.43616488575935364},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.2387862205505371}],"concepts":[{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.8942902088165283},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8375104665756226},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.7317570447921753},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5476871132850647},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5340332984924316},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.4817497730255127},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.45513689517974854},{"id":"https://openalex.org/C2776230583","wikidata":"https://www.wikidata.org/wiki/Q1322198","display_name":"Spoken language","level":2,"score":0.44543343782424927},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.43616488575935364},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.2387862205505371}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp48485.2024.10446888","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10446888","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W68733909","https://openalex.org/W1861492603","https://openalex.org/W1905882502","https://openalex.org/W1956340063","https://openalex.org/W2109586012","https://openalex.org/W2133459682","https://openalex.org/W2154652894","https://openalex.org/W2506483933","https://openalex.org/W2586148577","https://openalex.org/W2752796333","https://openalex.org/W2886641317","https://openalex.org/W2962862718","https://openalex.org/W2972394484","https://openalex.org/W3024605872","https://openalex.org/W3036601975","https://openalex.org/W3037465386","https://openalex.org/W3094502228","https://openalex.org/W3099142230","https://openalex.org/W3140429000","https://openalex.org/W3155217823","https://openalex.org/W3174311593","https://openalex.org/W3180355996","https://openalex.org/W3206387123","https://openalex.org/W3209059054","https://openalex.org/W4206865574","https://openalex.org/W4213019189","https://openalex.org/W4287854499","https://openalex.org/W4296070387","https://openalex.org/W4307680525","https://openalex.org/W4319862477","https://openalex.org/W4320458302","https://openalex.org/W4372260534","https://openalex.org/W4375868850","https://openalex.org/W4375868953","https://openalex.org/W4385245566","https://openalex.org/W4385570550","https://openalex.org/W4385822683","https://openalex.org/W4385823403","https://openalex.org/W4385970143","https://openalex.org/W4386071467","https://openalex.org/W4390871839","https://openalex.org/W4390874021","https://openalex.org/W4394671563","https://openalex.org/W6630875275","https://openalex.org/W6676497082","https://openalex.org/W6677929280","https://openalex.org/W6682631176","https://openalex.org/W6777028661","https://openalex.org/W6780218876","https://openalex.org/W6783867762","https://openalex.org/W6790356757","https://openalex.org/W6791353385","https://openalex.org/W6796464841","https://openalex.org/W6802517614","https://openalex.org/W6807079506","https://openalex.org/W6850204008","https://openalex.org/W6855650468","https://openalex.org/W6898505805"],"related_works":["https://openalex.org/W4210416330","https://openalex.org/W2775506363","https://openalex.org/W3088136942","https://openalex.org/W4290852288","https://openalex.org/W2949362007","https://openalex.org/W4283207562","https://openalex.org/W2963177403","https://openalex.org/W2330246314","https://openalex.org/W2949522393","https://openalex.org/W4289422896"],"abstract_inverted_index":{"In":[0],"this":[1,17],"paper,":[2],"we":[3,19,102,118,133,155],"propose":[4],"methods":[5],"to":[6,27,79,128,167,172],"build":[7],"a":[8,34,59],"powerful":[9],"and":[10,30,115],"efficient":[11],"Image-to-Speech":[12],"captioning":[13],"(Im2Sp)":[14],"model.":[15,62,126],"To":[16],"end,":[18],"start":[20],"with":[21],"importing":[22],"the":[23,43,46,54,81,86,91,98,121,124,129,135,148,159,173],"rich":[24],"knowledge":[25],"related":[26],"image":[28,137,139,153,165,175],"comprehension":[29],"language":[31,82,93],"modeling":[32,83,94],"from":[33],"large-scale":[35],"pre-trained":[36,87],"vision-language":[37,88,99],"model":[38,89],"into":[39,90,138],"Im2Sp.":[40,96],"We":[41],"set":[42,103],"output":[44],"of":[45,58,74,85,95,123,147,179],"proposed":[47],"Im2Sp":[48,106,125],"as":[49],"discretized":[50],"speech":[51,56,61,64,130],"units,":[52,140,154],"i.e.,":[53],"quantized":[55],"features":[57],"self-supervised":[60],"The":[63],"units":[65],"mainly":[66],"contain":[67],"linguistic":[68],"information":[69],"while":[70],"suppressing":[71],"other":[72],"characteristics":[73],"speech.":[75],"This":[76],"allows":[77],"us":[78],"incorporate":[80],"capability":[84],"spoken":[92],"With":[97,151],"pre-training":[100],"strategy,":[101],"new":[104],"state-of-the-art":[105],"performances":[107],"on":[108],"two":[109],"widely":[110],"used":[111],"benchmark":[112],"databases,":[113],"COCO":[114],"Flickr8k.":[116],"Then,":[117],"further":[119],"improve":[120],"efficiency":[122],"Similar":[127],"unit":[131],"case,":[132],"convert":[134],"original":[136,174],"which":[141],"are":[142],"derived":[143],"through":[144],"vector":[145],"quantization":[146],"raw":[149],"image.":[150],"these":[152],"can":[156],"drastically":[157],"reduce":[158],"required":[160],"data":[161,166,176],"storage":[162],"for":[163],"saving":[164],"just":[168],"0.8%":[169],"when":[170],"compared":[171],"in":[177],"terms":[178],"bits.":[180],"Demo":[181],"page:":[182],"bit.ly/3Z9T6LJ.":[183]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}