{"id":"https://openalex.org/W3034875620","doi":"https://doi.org/10.21437/interspeech.2021-1312","title":"AVLnet: Learning Audio-Visual Language Representations from Instructional Videos","display_name":"AVLnet: Learning Audio-Visual Language Representations from Instructional Videos","alternate_titles":[],"publication_year":2021,"publication_date":"2021-08-30","ids":{"openalex":"https://openalex.org/W3034875620","doi":"https://doi.org/10.21437/interspeech.2021-1312","mag":"3034875620"},"language":"en","primary_location":{"id":"doi:10.21437/interspeech.2021-1312","doi":"https://doi.org/10.21437/interspeech.2021-1312","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2021-1312","pdf_url":null,"source":{"id":"https://openalex.org/S4393918409","display_name":"Interspeech (USB)","issn_l":"2308-457X","issn":["2308-457X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-1","norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2021","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W3034875620"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2006.09199","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010091252","display_name":"Andrew Rouditchenko","orcid":"https://orcid.org/0000-0002-0063-3612","observed_orcids":["https://orcid.org/0000-0002-0063-3612"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Andrew Rouditchenko","raw_affiliation_strings":["Massachusetts Institute Of Technology#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute Of Technology#TAB#","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086597264","display_name":"Angie Boggust","orcid":"https://orcid.org/0000-0002-9671-5574","observed_orcids":["https://orcid.org/0000-0002-9671-5574"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Angie Boggust","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004717608","display_name":"David F. Harwath","orcid":"https://orcid.org/0000-0003-0206-0253","observed_orcids":["https://orcid.org/0000-0003-0206-0253"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"David Harwath","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Brian Chen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Brian Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103960169","display_name":"Dhiraj Joshi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Dhiraj Joshi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101787514","display_name":"Samuel Thomas","orcid":"https://orcid.org/0000-0001-7573-0620","observed_orcids":["https://orcid.org/0000-0001-7573-0620"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Samuel Thomas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015927589","display_name":"Kartik Audhkhasi","orcid":"https://orcid.org/0000-0002-2340-1144","observed_orcids":["https://orcid.org/0000-0002-2340-1144"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Kartik Audhkhasi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011795407","display_name":"Hilde Kuehne","orcid":"https://orcid.org/0000-0003-1079-4441","observed_orcids":["https://orcid.org/0000-0003-1079-4441"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Hilde Kuehne","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Rameswar Panda","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Rameswar Panda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052325109","display_name":"Rog\u00e9rio Feris","orcid":"https://orcid.org/0000-0001-6399-0679","observed_orcids":["https://orcid.org/0000-0001-6399-0679"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Rogerio Feris","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003725957","display_name":"Brian E. D. Kingsbury","orcid":"https://orcid.org/0000-0002-1343-6837","observed_orcids":["https://orcid.org/0000-0002-1343-6837"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Brian Kingsbury","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034529775","display_name":"Michael Picheny","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Michael Picheny","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085020955","display_name":"Antonio B. Torralba","orcid":"https://orcid.org/0000-0003-4915-0256","observed_orcids":["https://orcid.org/0000-0003-4915-0256"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Antonio Torralba","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5112758056","display_name":"James Robert Glass","orcid":"https://orcid.org/0000-0002-3097-360X","observed_orcids":["https://orcid.org/0000-0002-3097-360X"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"James Glass","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.5087,"has_fulltext":true,"cited_by_count":160,"citation_normalized_percentile":{"value":0.982768,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1584","last_page":"1588"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/self-supervised-learning","display_name":"self-supervised learning","score":0.8410000205039978},{"id":"https://openalex.org/keywords/educational-videos","display_name":"educational videos","score":0.921999990940094},{"id":"https://openalex.org/keywords/image-retrieval","display_name":"image retrieval","score":0.6629999876022339},{"id":"https://openalex.org/keywords/video-retrieval","display_name":"video retrieval","score":0.9980000257492065},{"id":"https://openalex.org/keywords/text-video-retrieval","display_name":"video-text retrieval","score":0.906000018119812},{"id":"https://openalex.org/keywords/multimodal-embeddings","display_name":"multimodal embeddings","score":0.49799999594688416}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8797540664672852},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5798476934432983},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5554625391960144},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5106707215309143},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5045667886734009},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.4623298645019531},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.45744818449020386},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4449724555015564},{"id":"https://openalex.org/C160372630","wikidata":"https://www.wikidata.org/wiki/Q4819855","display_name":"Audio analyzer","level":5,"score":0.42714935541152954},{"id":"https://openalex.org/C127220857","wikidata":"https://www.wikidata.org/wiki/Q2719318","display_name":"Audio signal processing","level":4,"score":0.2674825191497803},{"id":"https://openalex.org/C64922751","wikidata":"https://www.wikidata.org/wiki/Q4650799","display_name":"Audio signal","level":3,"score":0.2318432629108429},{"id":"https://openalex.org/C13895895","wikidata":"https://www.wikidata.org/wiki/Q3270773","display_name":"Speech coding","level":2,"score":0.10837686061859131},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.21437/interspeech.2021-1312","doi":"https://doi.org/10.21437/interspeech.2021-1312","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2021-1312","pdf_url":null,"source":{"id":"https://openalex.org/S4393918409","display_name":"Interspeech (USB)","issn_l":"2308-457X","issn":["2308-457X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-1","norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2021","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2006.09199","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2006.09199","pdf_url":"https://arxiv.org/pdf/2006.09199","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3034875620","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/pdf/2006.09199.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2006.09199","doi":"https://doi.org/10.48550/arxiv.2006.09199","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2006.09199","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2006.09199","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2006.09199","pdf_url":"https://arxiv.org/pdf/2006.09199","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3034875620.pdf","grobid_xml":"https://content.openalex.org/works/W3034875620.grobid-xml"},"referenced_works_count":97,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1614298861","https://openalex.org/W1899185266","https://openalex.org/W2108598243","https://openalex.org/W2134670479","https://openalex.org/W2137010615","https://openalex.org/W2152790380","https://openalex.org/W2185243164","https://openalex.org/W2194775991","https://openalex.org/W2196697617","https://openalex.org/W2259472270","https://openalex.org/W2425121537","https://openalex.org/W2511428026","https://openalex.org/W2544224704","https://openalex.org/W2556930864","https://openalex.org/W2580178245","https://openalex.org/W2586148577","https://openalex.org/W2592749667","https://openalex.org/W2619082050","https://openalex.org/W2619697695","https://openalex.org/W2706729717","https://openalex.org/W2736876693","https://openalex.org/W2773824886","https://openalex.org/W2787779284","https://openalex.org/W2795187948","https://openalex.org/W2796207103","https://openalex.org/W2796292145","https://openalex.org/W2796315435","https://openalex.org/W2796992393","https://openalex.org/W2798708692","https://openalex.org/W2803088946","https://openalex.org/W2805122419","https://openalex.org/W2808286951","https://openalex.org/W2808399042","https://openalex.org/W2885775891","https://openalex.org/W2890052321","https://openalex.org/W2890855852","https://openalex.org/W2899274165","https://openalex.org/W2899922369","https://openalex.org/W2901488116","https://openalex.org/W2903320905","https://openalex.org/W2914699769","https://openalex.org/W2923016229","https://openalex.org/W2935917634","https://openalex.org/W2936229313","https://openalex.org/W2938991416","https://openalex.org/W2939598386","https://openalex.org/W2942642798","https://openalex.org/W2948242301","https://openalex.org/W2948721154","https://openalex.org/W2948859046","https://openalex.org/W2952132648","https://openalex.org/W2953114965","https://openalex.org/W2962865004","https://openalex.org/W2962918445","https://openalex.org/W2962934715","https://openalex.org/W2963115079","https://openalex.org/W2963218389","https://openalex.org/W2963807156","https://openalex.org/W2963902314","https://openalex.org/W2963996492","https://openalex.org/W2964037671","https://openalex.org/W2964094654","https://openalex.org/W2964121744","https://openalex.org/W2965147078","https://openalex.org/W2965458216","https://openalex.org/W2968848930","https://openalex.org/W2970971581","https://openalex.org/W2971709506","https://openalex.org/W2973009197","https://openalex.org/W2973132572","https://openalex.org/W2974393448","https://openalex.org/W2975357369","https://openalex.org/W2981851019","https://openalex.org/W2982624843","https://openalex.org/W2988200020","https://openalex.org/W2989980422","https://openalex.org/W2990408345","https://openalex.org/W2991557631","https://openalex.org/W3005680577","https://openalex.org/W3009380496","https://openalex.org/W3034742263","https://openalex.org/W3034978746","https://openalex.org/W3035635319","https://openalex.org/W3095293218","https://openalex.org/W3096372900","https://openalex.org/W3100177202","https://openalex.org/W3108380215","https://openalex.org/W3157861865","https://openalex.org/W3160014322","https://openalex.org/W3161204797","https://openalex.org/W3161945002","https://openalex.org/W3176398504","https://openalex.org/W3196698946","https://openalex.org/W4239072543","https://openalex.org/W4297808394","https://openalex.org/W4394453761"],"related_works":["https://openalex.org/W2619697695","https://openalex.org/W3196694757","https://openalex.org/W4321786075","https://openalex.org/W4396913488","https://openalex.org/W7067044464","https://openalex.org/W7140237245","https://openalex.org/W4392132288","https://openalex.org/W7169790875","https://openalex.org/W3136482419","https://openalex.org/W4415433442"],"abstract_inverted_index":{"Current":[0],"methods":[1],"for":[2,56],"learning":[3],"visually":[4],"grounded":[5],"language":[6],"from":[7,48,63],"videos":[8],"often":[9],"rely":[10],"on":[11,76,88],"text":[12,57],"annotation,":[13,58],"such":[14],"as":[15],"human":[16],"generated":[17,21],"captions":[18],"or":[19],"machine":[20],"automatic":[22],"speech":[23,110],"recognition":[24],"(ASR)":[25],"transcripts.":[26],"In":[27],"this":[28],"work,":[29],"we":[30,59,99],"introduce":[31],"the":[32,54],"Audio-Video":[33],"Language":[34],"Network":[35],"(AVLnet),":[36],"a":[37,42,78],"self-supervised":[38],"network":[39],"that":[40],"learns":[41],"shared":[43],"audio-visual":[44,61,116],"embedding":[45],"space":[46],"directly":[47],"raw":[49,70],"video":[50,66,92],"inputs.":[51],"To":[52],"circumvent":[53],"need":[55],"learn":[60,115],"representations":[62],"randomly":[64],"segmented":[65],"clips":[67],"and":[68,86,91,111],"their":[69],"audio":[71],"waveforms.":[72],"We":[73],"train":[74],"AVLnet":[75],"HowTo100M,":[77],"large":[79],"corpus":[80],"of":[81,102],"publicly":[82],"available":[83],"instructional":[84],"videos,":[85],"evaluate":[87],"image":[89],"retrieval":[90,93],"tasks,":[94],"achieving":[95],"state-of-the-art":[96],"performance.":[97],"Finally,":[98],"perform":[100],"analysis":[101],"AVLnet\u2019s":[103],"learned":[104],"representations,":[105],"showing":[106],"our":[107],"model":[108],"utilizes":[109],"natural":[112],"sounds":[113],"to":[114],"concepts.":[117]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3034875620","counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":19},{"year":2024,"cited_by_count":26},{"year":2023,"cited_by_count":40},{"year":2022,"cited_by_count":40},{"year":2021,"cited_by_count":24},{"year":2020,"cited_by_count":4}],"updated_date":"2026-11-21T10:27:49.728292","created_date":"2020-06-19T00:00:00"}
