{"id":"https://openalex.org/W3095720143","doi":"https://doi.org/10.21437/interspeech.2020-3157","title":"Caption Alignment for Low Resource Audio-Visual Data","display_name":"Caption Alignment for Low Resource Audio-Visual Data","publication_year":2020,"publication_date":"2020-10-25","ids":{"openalex":"https://openalex.org/W3095720143","doi":"https://doi.org/10.21437/interspeech.2020-3157","mag":"3095720143"},"language":"en","primary_location":{"id":"doi:10.21437/interspeech.2020-3157","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2020-3157","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2020","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://research.monash.edu/en/publications/be5d5cf9-f128-4515-9390-3457b7231720","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5025345539","display_name":"Vighnesh Reddy Konda","orcid":null},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Vighnesh Reddy Konda","raw_affiliation_strings":["Indian Institute of Technology Bombay, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology Bombay, India","institution_ids":["https://openalex.org/I162827531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008338781","display_name":"Mayur Warialani","orcid":null},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Mayur Warialani","raw_affiliation_strings":["Indian Institute of Technology Bombay, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology Bombay, India","institution_ids":["https://openalex.org/I162827531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008282727","display_name":"Rakesh Prasanth Achari","orcid":null},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Rakesh Prasanth Achari","raw_affiliation_strings":["Indian Institute of Technology Bombay, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology Bombay, India","institution_ids":["https://openalex.org/I162827531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022870639","display_name":"Varad Bhatnagar","orcid":null},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Varad Bhatnagar","raw_affiliation_strings":["Indian Institute of Technology Bombay, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology Bombay, India","institution_ids":["https://openalex.org/I162827531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064447033","display_name":"Jayaprakash Akula","orcid":"https://orcid.org/0000-0002-1612-1064"},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Jayaprakash Akula","raw_affiliation_strings":["Indian Institute of Technology Bombay, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology Bombay, India","institution_ids":["https://openalex.org/I162827531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036738038","display_name":"Preethi Jyothi","orcid":null},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Preethi Jyothi","raw_affiliation_strings":["Indian Institute of Technology Bombay, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology Bombay, India","institution_ids":["https://openalex.org/I162827531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089606464","display_name":"Ganesh Ramakrishnan","orcid":"https://orcid.org/0000-0003-4533-2490"},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Ganesh Ramakrishnan","raw_affiliation_strings":["Indian Institute of Technology Bombay, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology Bombay, India","institution_ids":["https://openalex.org/I162827531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081525024","display_name":"Gholamreza Haffari","orcid":"https://orcid.org/0000-0001-7326-8380"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Gholamreza Haffari","raw_affiliation_strings":["Monash University, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Monash University, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5049440381","display_name":"Pankaj Pratap Singh","orcid":"https://orcid.org/0000-0003-4079-4485"},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Pankaj Singh","raw_affiliation_strings":["Indian Institute of Technology Bombay, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology Bombay, India","institution_ids":["https://openalex.org/I162827531"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":9,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.12405427,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3525","last_page":"3529"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.833171546459198},{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.7536762952804565},{"id":"https://openalex.org/keywords/search-engine-indexing","display_name":"Search engine indexing","score":0.6022731065750122},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.5305184721946716},{"id":"https://openalex.org/keywords/resource","display_name":"Resource (disambiguation)","score":0.4976551830768585},{"id":"https://openalex.org/keywords/audio-visual","display_name":"Audio visual","score":0.4791554808616638},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.47686514258384705},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4626382887363434},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.45587342977523804},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.45524802803993225},{"id":"https://openalex.org/keywords/audio-signal","display_name":"Audio signal","score":0.4373214840888977},{"id":"https://openalex.org/keywords/audio-signal-processing","display_name":"Audio signal processing","score":0.41000378131866455},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3252567648887634},{"id":"https://openalex.org/keywords/speech-coding","display_name":"Speech coding","score":0.1425900161266327}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.833171546459198},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.7536762952804565},{"id":"https://openalex.org/C75165309","wikidata":"https://www.wikidata.org/wiki/Q2258979","display_name":"Search engine indexing","level":2,"score":0.6022731065750122},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.5305184721946716},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.4976551830768585},{"id":"https://openalex.org/C3017588708","wikidata":"https://www.wikidata.org/wiki/Q758901","display_name":"Audio visual","level":2,"score":0.4791554808616638},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.47686514258384705},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4626382887363434},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.45587342977523804},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.45524802803993225},{"id":"https://openalex.org/C64922751","wikidata":"https://www.wikidata.org/wiki/Q4650799","display_name":"Audio signal","level":3,"score":0.4373214840888977},{"id":"https://openalex.org/C127220857","wikidata":"https://www.wikidata.org/wiki/Q2719318","display_name":"Audio signal processing","level":4,"score":0.41000378131866455},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3252567648887634},{"id":"https://openalex.org/C13895895","wikidata":"https://www.wikidata.org/wiki/Q3270773","display_name":"Speech coding","level":2,"score":0.1425900161266327},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.21437/interspeech.2020-3157","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2020-3157","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2020","raw_type":"proceedings-article"},{"id":"pmh:oai:monash.edu:openaire/be5d5cf9-f128-4515-9390-3457b7231720","is_oa":true,"landing_page_url":"https://research.monash.edu/en/publications/be5d5cf9-f128-4515-9390-3457b7231720","pdf_url":null,"source":{"id":"https://openalex.org/S4306402625","display_name":"Monash University Research Portal (Monash University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I56590836","host_organization_name":"Monash University","host_organization_lineage":["https://openalex.org/I56590836"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Konda, V R, Warialani, M, Achari, R P, Bhatnagar, V, Akula, J, Jyothi, P, Ramakrishnan, G, Haffari, G & Singh, P 2020, Caption alignment for low resource audio-visual data. in H Meng (ed.), Proceedings of the Annual Conference of the International Speech Communication Association, INTERSPEECH. vol. 2020-October, International Speech Communication Association (ISCA), Baixas FRANCE, pp. 3525-3529, Annual Conference of the International Speech Communication Association (was Eurospeech) 2020, Shanghai, China, 25/10/20. https://doi.org/10.21437/Interspeech.2020-3157","raw_type":"contributionToPeriodical"},{"id":"pmh:oai:monash.edu:publications/be5d5cf9-f128-4515-9390-3457b7231720","is_oa":true,"landing_page_url":"http://www.scopus.com/inward/record.url?scp=85098150159&partnerID=8YFLogxK","pdf_url":null,"source":{"id":"https://openalex.org/S4306402625","display_name":"Monash University Research Portal (Monash University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I56590836","host_organization_name":"Monash University","host_organization_lineage":["https://openalex.org/I56590836"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Konda , V R , Warialani , M , Achari , R P , Bhatnagar , V , Akula , J , Jyothi , P , Ramakrishnan , G , Haffari , G & Singh , P 2020 , Caption alignment for low resource audio-visual data . in H Meng (ed.) , Proceedings of the Annual Conference of the International Speech Communication Association, INTERSPEECH . vol. 2020-October , International Speech Communication Association (ISCA) , Baixas FRANCE , pp. 3525-3529 , Annual Conference of the International Speech Communication Association (was Eurospeech) 2020 , Shanghai , China , 25/10/20 . https://doi.org/10.21437/Interspeech.2020-3157","raw_type":"contributionToPeriodical"}],"best_oa_location":{"id":"pmh:oai:monash.edu:openaire/be5d5cf9-f128-4515-9390-3457b7231720","is_oa":true,"landing_page_url":"https://research.monash.edu/en/publications/be5d5cf9-f128-4515-9390-3457b7231720","pdf_url":null,"source":{"id":"https://openalex.org/S4306402625","display_name":"Monash University Research Portal (Monash University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I56590836","host_organization_name":"Monash University","host_organization_lineage":["https://openalex.org/I56590836"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Konda, V R, Warialani, M, Achari, R P, Bhatnagar, V, Akula, J, Jyothi, P, Ramakrishnan, G, Haffari, G & Singh, P 2020, Caption alignment for low resource audio-visual data. in H Meng (ed.), Proceedings of the Annual Conference of the International Speech Communication Association, INTERSPEECH. vol. 2020-October, International Speech Communication Association (ISCA), Baixas FRANCE, pp. 3525-3529, Annual Conference of the International Speech Communication Association (was Eurospeech) 2020, Shanghai, China, 25/10/20. https://doi.org/10.21437/Interspeech.2020-3157","raw_type":"contributionToPeriodical"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.4300000071525574}],"awards":[],"funders":[{"id":"https://openalex.org/F4320323448","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W1524333225","https://openalex.org/W1586939924","https://openalex.org/W1983364832","https://openalex.org/W2052384514","https://openalex.org/W2493916176","https://openalex.org/W2515666985","https://openalex.org/W2523993696","https://openalex.org/W2526050071","https://openalex.org/W2527349934","https://openalex.org/W2584992898","https://openalex.org/W2593116425","https://openalex.org/W2739107216","https://openalex.org/W2766375149","https://openalex.org/W2785892019","https://openalex.org/W2789221157","https://openalex.org/W2798354744","https://openalex.org/W2808647806","https://openalex.org/W2890502146","https://openalex.org/W2891456603","https://openalex.org/W2897628926","https://openalex.org/W2962869524","https://openalex.org/W2963017553","https://openalex.org/W2963252191","https://openalex.org/W2963393391","https://openalex.org/W2963916161","https://openalex.org/W2964045146","https://openalex.org/W2964089981","https://openalex.org/W2964232540","https://openalex.org/W3002741552","https://openalex.org/W3015213852"],"related_works":["https://openalex.org/W2289868279","https://openalex.org/W1479881079","https://openalex.org/W2970176078","https://openalex.org/W4231351862","https://openalex.org/W4212794605","https://openalex.org/W4315836293","https://openalex.org/W4243888788","https://openalex.org/W1975359510","https://openalex.org/W3004352674","https://openalex.org/W2769884427"],"abstract_inverted_index":{"Understanding":[0],"videos":[1],"via":[2],"captioning":[3],"has":[4,45,92],"gained":[5],"a":[6,21,25,87],"lot":[7],"of":[8,62,74,81,138,152],"traction":[9],"recently.":[10],"While":[11],"captions":[12,82,93,118],"are":[13],"provided":[14],"alongside":[15],"videos,":[16],"the":[17,52,60,63,75,100,104,110,150],"information":[18],"about":[19],"where":[20],"caption":[22],"aligns":[23],"within":[24,83],"video":[26,142],"is":[27,114],"missing,":[28],"which":[29],"could":[30],"be":[31],"particularly":[32],"useful":[33],"for":[34,78,135,143],"indexing":[35],"and":[36,50,102,106,116,123,141,158,161],"retrieval.":[37],"Existing":[38],"work":[39],"on":[40,68],"learning":[41],"to":[42,70,99,174],"infer":[43],"alignments":[44],"mostly":[46],"exploited":[47],"visual":[48],"features":[49],"ignored":[51],"audio":[53,64,76,101,140,153,178],"signal.":[54],"Video":[55],"understanding":[56],"applications":[57],"often":[58],"underestimate":[59],"importance":[61],"modality.":[65],"We":[66,85,128,146],"focus":[67],"how":[69],"make":[71],"effective":[72,136,163],"use":[73],"modality":[77],"temporal":[79,144],"localization":[80],"videos.":[84],"release":[86],"new":[88],"audio-visual":[89],"dataset":[90,113],"that":[91],"time-aligned":[94],"by":[95],"(i)":[96,149],"carefully":[97],"listening":[98],"watching":[103,108],"video,":[105],"(ii)":[107,162],"only":[109],"video.":[111],"Our":[112],"audio-rich":[115],"contains":[117],"in":[119,154],"two":[120],"languages,":[121],"English":[122],"Marathi":[124],"(a":[125],"low-resource":[126,172],"language).":[127],"further":[129],"propose":[130],"an":[131],"attention-driven":[132],"multimodal":[133],"model,":[134],"utilization":[137],"both":[139,155],"localization.":[145],"then":[147],"investigate":[148],"effects":[151],"data":[156],"preparation":[157],"model":[159],"design,":[160],"pretraining":[164],"strategies":[165],"(Audioset,":[166],"ASR-bottleneck":[167],"features,":[168],"PASE,":[169],"etc.)":[170],"handling":[171],"setting":[173],"help":[175],"extract":[176],"rich":[177],"representations.":[179]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}