{"id":"https://openalex.org/W2809716557","doi":null,"title":"Co-Training of Audio and Video Representations from Self-Supervised Temporal Synchronization","display_name":"Co-Training of Audio and Video Representations from Self-Supervised Temporal Synchronization","publication_year":2018,"publication_date":"2018-06-30","ids":{"openalex":"https://openalex.org/W2809716557","mag":"2809716557"},"language":"en","primary_location":{"id":"mag:2809716557","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1807.00230v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/abs/1807.00230v1","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070413665","display_name":"Bruno Korbar","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bruno Korbar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016319243","display_name":"Du Tran","orcid":"https://orcid.org/0000-0001-9673-7194","observed_orcids":["https://orcid.org/0000-0001-9673-7194"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du Tran","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-9673-7194","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5082736347","display_name":"Lorenzo Torresani","orcid":"https://orcid.org/0000-0002-6924-7285","observed_orcids":["https://orcid.org/0000-0002-6924-7285"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lorenzo Torresani","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":44,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.5952000021934509,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.5952000021934509,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.10620000213384628,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.08659999817609787,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/audio-visual-synchronization","display_name":"audio-video synchronization","score":0.8029999732971191},{"id":"https://openalex.org/keywords/self-supervised-learning","display_name":"self-supervised learning","score":0.8519999980926514},{"id":"https://openalex.org/keywords/co-training","display_name":"co-training","score":0.515999972820282},{"id":"https://openalex.org/keywords/curriculum-learning","display_name":"curriculum learning","score":0.7670000195503235},{"id":"https://openalex.org/keywords/contrastive-loss","display_name":"contrastive loss","score":0.8880000114440918},{"id":"https://openalex.org/keywords/negative-example-selection","display_name":"negative example selection","score":0.7300000190734863},{"id":"https://openalex.org/keywords/audio-classification","display_name":"audio classification","score":0.8939999938011169},{"id":"https://openalex.org/keywords/video-action-recognition","display_name":"video action recognition","score":0.9120000004768372},{"id":"https://openalex.org/keywords/pretraining","display_name":"pre-training","score":0.4620000123977661},{"id":"https://openalex.org/keywords/temporal-synchronization","display_name":"temporal synchronization","score":0.5019999742507935}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8180508017539978},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.7256216406822205},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6560297012329102},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.5936576128005981},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5471292734146118},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5305818915367126},{"id":"https://openalex.org/C2987834672","wikidata":"https://www.wikidata.org/wiki/Q4677630","display_name":"Action recognition","level":3,"score":0.4767219126224518},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.38658928871154785},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3605807423591614},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.08504387736320496},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:2809716557","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1807.00230v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:2809716557","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1807.00230v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W24089286","https://openalex.org/W219040644","https://openalex.org/W639708223","https://openalex.org/W1903029394","https://openalex.org/W1927052826","https://openalex.org/W1999192586","https://openalex.org/W2105101328","https://openalex.org/W2108598243","https://openalex.org/W2110798204","https://openalex.org/W2126579184","https://openalex.org/W2286929393","https://openalex.org/W2296073425","https://openalex.org/W2511428026","https://openalex.org/W2544224704","https://openalex.org/W2550462002","https://openalex.org/W2553594924","https://openalex.org/W2593116425","https://openalex.org/W2604379605","https://openalex.org/W2619082050","https://openalex.org/W2619947201","https://openalex.org/W2745357708","https://openalex.org/W2796992393","https://openalex.org/W2797263747","https://openalex.org/W2950187998","https://openalex.org/W2950789693","https://openalex.org/W2951874610","https://openalex.org/W3091905774"],"related_works":["https://openalex.org/W2890052321","https://openalex.org/W4382237564","https://openalex.org/W3048939150","https://openalex.org/W2796992393","https://openalex.org/W3020933450","https://openalex.org/W3214261769","https://openalex.org/W3095687795","https://openalex.org/W3173388686","https://openalex.org/W2950864153","https://openalex.org/W4382318126"],"abstract_inverted_index":{"There":[0],"is":[1],"a":[2,12,40,45,55,107,129,141],"natural":[3],"correlation":[4],"between":[5],"the":[6,52,79,89,100,113],"visual":[7,104],"and":[8,24,30,51,97,140],"auditive":[9],"elements":[10],"of":[11,48,54,73,115,132,143],"video.":[13],"In":[14],"this":[15,19],"work":[16],"we":[17],"leverage":[18],"connection":[20],"to":[21,61,69,88,111,121],"learn":[22],"general":[23],"effective":[25,109],"features":[26,82],"for":[27],"both":[28],"audio":[29,81,93],"video":[31],"analysis":[32],"from":[33,66,123],"self-supervised":[34,126],"temporal":[35,71],"synchronization.":[36],"We":[37],"demonstrate":[38],"that":[39],"calibrated":[41],"curriculum":[42],"learning":[43,122],"scheme,":[44],"careful":[46],"choice":[47],"negative":[49],"examples,":[50],"use":[53],"contrastive":[56],"loss":[57],"are":[58],"critical":[59],"ingredients":[60],"obtain":[62],"powerful":[63],"multi-sensory":[64],"representations":[65],"models":[67],"optimized":[68],"discern":[70],"synchronization":[72],"audio-video":[74],"pairs.":[75],"Without":[76],"further":[77],"finetuning,":[78],"resulting":[80],"achieve":[83],"performance":[84],"superior":[85],"or":[86],"comparable":[87],"state-of-the-art":[90],"on":[91,138,145],"established":[92],"classification":[94],"benchmarks":[95],"(DCASE2014":[96],"ESC-50).":[98],"At":[99],"same":[101],"time,":[102],"our":[103,125],"subnet":[105],"provides":[106],"very":[108],"initialization":[110],"improve":[112],"accuracy":[114,137],"video-based":[116],"action":[117,135],"recognition":[118,136],"models:":[119],"compared":[120],"scratch,":[124],"pretraining":[127],"yields":[128],"remarkable":[130],"gain":[131],"+16.7%":[133],"in":[134],"UCF101":[139],"boost":[142],"+13.0%":[144],"HMDB51.":[146]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":12},{"year":2019,"cited_by_count":8},{"year":2018,"cited_by_count":2}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2018-07-10T00:00:00"}