{"id":"https://openalex.org/W4285110637","doi":"https://doi.org/10.1109/jstsp.2022.3180220","title":"Keyword Localisation in Untranscribed Speech Using Visually Grounded Speech Models","display_name":"Keyword Localisation in Untranscribed Speech Using Visually Grounded Speech Models","publication_year":2022,"publication_date":"2022-06-03","ids":{"openalex":"https://openalex.org/W4285110637","doi":"https://doi.org/10.1109/jstsp.2022.3180220"},"language":"en","primary_location":{"id":"doi:10.1109/jstsp.2022.3180220","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2022.3180220","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081862617","display_name":"Kayode Olaleye","orcid":"https://orcid.org/0000-0002-2622-7305"},"institutions":[{"id":"https://openalex.org/I26092322","display_name":"Stellenbosch University","ror":"https://ror.org/05bk57929","country_code":"ZA","type":"education","lineage":["https://openalex.org/I26092322"]}],"countries":["ZA"],"is_corresponding":false,"raw_author_name":"Kayode Olaleye","raw_affiliation_strings":["Stellenbosch University, Stellenbosch, South Africa"],"raw_orcid":"https://orcid.org/0000-0002-2622-7305","affiliations":[{"raw_affiliation_string":"Stellenbosch University, Stellenbosch, South Africa","institution_ids":["https://openalex.org/I26092322"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013389452","display_name":"Dan Onea\u0163\u0103","orcid":"https://orcid.org/0000-0003-4354-4393"},"institutions":[{"id":"https://openalex.org/I61641377","display_name":"Universitatea Na\u021bional\u0103 de \u0218tiin\u021b\u0103 \u0219i Tehnologie Politehnica Bucure\u0219ti","ror":"https://ror.org/0558j5q12","country_code":"RO","type":"education","lineage":["https://openalex.org/I61641377"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Dan Oneata","raw_affiliation_strings":["University Politehnica of Bucharest, Bucharest, Romania"],"raw_orcid":"https://orcid.org/0000-0003-4354-4393","affiliations":[{"raw_affiliation_string":"University Politehnica of Bucharest, Bucharest, Romania","institution_ids":["https://openalex.org/I61641377"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040305929","display_name":"Herman Kamper","orcid":"https://orcid.org/0000-0003-2980-3475"},"institutions":[{"id":"https://openalex.org/I26092322","display_name":"Stellenbosch University","ror":"https://ror.org/05bk57929","country_code":"ZA","type":"education","lineage":["https://openalex.org/I26092322"]}],"countries":["ZA"],"is_corresponding":false,"raw_author_name":"Herman Kamper","raw_affiliation_strings":["Stellenbosch University, Stellenbosch, South Africa"],"raw_orcid":"https://orcid.org/0000-0003-2980-3475","affiliations":[{"raw_affiliation_string":"Stellenbosch University, Stellenbosch, South Africa","institution_ids":["https://openalex.org/I26092322"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2195,"currency":"USD","value_usd":2195},"apc_paid":null,"fwci":0.58,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.63311239,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":97},"biblio":{"volume":"16","issue":"6","first_page":"1454","last_page":"1466"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/utterance","display_name":"Utterance","score":0.7960038185119629},{"id":"https://openalex.org/keywords/keyword-spotting","display_name":"Keyword spotting","score":0.7917194962501526},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.77662593126297},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6119557619094849},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.5855666399002075},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5431709289550781},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5227078199386597},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.4833095073699951},{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.42890626192092896},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3799116015434265}],"concepts":[{"id":"https://openalex.org/C2775852435","wikidata":"https://www.wikidata.org/wiki/Q258403","display_name":"Utterance","level":2,"score":0.7960038185119629},{"id":"https://openalex.org/C2781213101","wikidata":"https://www.wikidata.org/wiki/Q6398558","display_name":"Keyword spotting","level":2,"score":0.7917194962501526},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.77662593126297},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6119557619094849},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.5855666399002075},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5431709289550781},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5227078199386597},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.4833095073699951},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.42890626192092896},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3799116015434265},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/jstsp.2022.3180220","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2022.3180220","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6600000262260437}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":110,"referenced_works":["https://openalex.org/W753847829","https://openalex.org/W1515241861","https://openalex.org/W1821462560","https://openalex.org/W1849277567","https://openalex.org/W1861492603","https://openalex.org/W1897761818","https://openalex.org/W1902237438","https://openalex.org/W1905882502","https://openalex.org/W1987959440","https://openalex.org/W1994488211","https://openalex.org/W2017814585","https://openalex.org/W2032943813","https://openalex.org/W2062955551","https://openalex.org/W2066134726","https://openalex.org/W2104247845","https://openalex.org/W2108598243","https://openalex.org/W2125165590","https://openalex.org/W2127716180","https://openalex.org/W2132921748","https://openalex.org/W2185175083","https://openalex.org/W2194775991","https://openalex.org/W2277195237","https://openalex.org/W2282219577","https://openalex.org/W2295107390","https://openalex.org/W2402146185","https://openalex.org/W2466918907","https://openalex.org/W2508907749","https://openalex.org/W2520859141","https://openalex.org/W2529337537","https://openalex.org/W2568262903","https://openalex.org/W2586148577","https://openalex.org/W2605131327","https://openalex.org/W2605432195","https://openalex.org/W2758849341","https://openalex.org/W2776207810","https://openalex.org/W2795151422","https://openalex.org/W2799062425","https://openalex.org/W2886300652","https://openalex.org/W2886641317","https://openalex.org/W2890704021","https://openalex.org/W2890964092","https://openalex.org/W2895651543","https://openalex.org/W2920166246","https://openalex.org/W2927673779","https://openalex.org/W2936774411","https://openalex.org/W2938991416","https://openalex.org/W2950133079","https://openalex.org/W2962735233","https://openalex.org/W2962753610","https://openalex.org/W2962858109","https://openalex.org/W2962862718","https://openalex.org/W2962969419","https://openalex.org/W2962980711","https://openalex.org/W2963330681","https://openalex.org/W2963630207","https://openalex.org/W2963902314","https://openalex.org/W2964001192","https://openalex.org/W2964115348","https://openalex.org/W2964249784","https://openalex.org/W2973135958","https://openalex.org/W2973180715","https://openalex.org/W2983256121","https://openalex.org/W2989358187","https://openalex.org/W3005578234","https://openalex.org/W3016087077","https://openalex.org/W3034368386","https://openalex.org/W3034873438","https://openalex.org/W3049010983","https://openalex.org/W3095670406","https://openalex.org/W3099142230","https://openalex.org/W3102219307","https://openalex.org/W3117408830","https://openalex.org/W3123798147","https://openalex.org/W3138154797","https://openalex.org/W3157861865","https://openalex.org/W3158986867","https://openalex.org/W3159476814","https://openalex.org/W3194502352","https://openalex.org/W3197828817","https://openalex.org/W3198411039","https://openalex.org/W3200287550","https://openalex.org/W3206387123","https://openalex.org/W4206075291","https://openalex.org/W4211190719","https://openalex.org/W4239171064","https://openalex.org/W4243159200","https://openalex.org/W4288083516","https://openalex.org/W4303633609","https://openalex.org/W6600827882","https://openalex.org/W6631190155","https://openalex.org/W6637373629","https://openalex.org/W6638523607","https://openalex.org/W6677994088","https://openalex.org/W6679909955","https://openalex.org/W6684821475","https://openalex.org/W6728167234","https://openalex.org/W6729831399","https://openalex.org/W6729977899","https://openalex.org/W6735703314","https://openalex.org/W6747701563","https://openalex.org/W6762718338","https://openalex.org/W6770596778","https://openalex.org/W6770805772","https://openalex.org/W6780294235","https://openalex.org/W6783797576","https://openalex.org/W6784333009","https://openalex.org/W6786915337","https://openalex.org/W6795140394","https://openalex.org/W6804270185","https://openalex.org/W6864391120"],"related_works":["https://openalex.org/W2114097550","https://openalex.org/W2918559346","https://openalex.org/W84309476","https://openalex.org/W4385352507","https://openalex.org/W4389912005","https://openalex.org/W4286904253","https://openalex.org/W2529301793","https://openalex.org/W2384121599","https://openalex.org/W2386245264","https://openalex.org/W2038083449"],"abstract_inverted_index":{"Keyword":[0],"localisation":[1,23,145,152,184,255],"is":[2,24,185,191,203,209,257],"the":[3,86,128,139,142,149,164,218,246,260,264,279],"task":[4],"of":[5,88,111,141,148,161],"finding":[6],"where":[7,183,197,273],"in":[8,92,171,194],"a":[9,12,27,71,76,81,89,132,155,168,181,195,198,206],"speech":[10,30],"utterance":[11,173],"given":[13],"query":[14],"keyword":[15,22,91,169,199,254],"occurs.":[16],"We":[17,99,267],"investigate":[18],"to":[19,84,103,120,176,217],"what":[20],"extent":[21],"possible":[25],"using":[26,70],"visually":[28],"grounded":[29],"(VGS)":[31],"model.":[32,143],"VGS":[33,82,105,156,227],"models":[34,45,106,228,242],"are":[35,46,214,243],"trained":[36],"on":[37],"unlabelled":[38],"images":[39,65],"paired":[40],"with":[41,66,75,107,158,221,259],"spoken":[42],"captions.":[43],"These":[44],"therefore":[47],"self-supervised\u2014trained":[48],"without":[49],"any":[50,232],"explicit":[51],"textual":[52,233],"label":[53],"or":[54,234,249],"location":[55,235],"information.":[56],"To":[57],"obtain":[58],"training":[59,64],"targets,":[60],"we":[61],"first":[62,204,247],"tag":[63],"soft":[67],"text":[68],"labels":[69],"pretrained":[72],"visual":[73,265],"classifier":[74],"fixed":[77],"vocabulary.":[78],"This":[79],"enables":[80],"model":[83,124,280],"predict":[85,177],"presence":[87],"written":[90],"an":[93,121,159,172],"utterance,":[94],"but":[95],"not":[96,230],"its":[97,178],"location.":[98,179],"consider":[100],"four":[101],"ways":[102],"equip":[104],"localisations":[108],"capabilities.":[109],"Two":[110],"these\u2014a":[112],"saliency":[113],"approach":[114],"and":[115,131,174,193,272],"input":[116],"masking\u2014can":[117],"be":[118],"applied":[119],"arbitrary":[122],"prediction":[123],"after":[125,187],"training,":[126],"while":[127],"other":[129],"two\u2014attention":[130],"score":[133],"aggregation":[134],"approach\u2014are":[135],"incorporated":[136],"directly":[137],"into":[138],"structure":[140],"Masked-based":[144],"gives":[146],"some":[147],"best":[150],"reported":[151],"scores":[153,213],"from":[154,263],"model,":[157],"accuracy":[160],"57%":[162],"when":[163],"system":[165],"knows":[166],"that":[167,240,278],"occurs":[170],"need":[175],"In":[180],"setting":[182,196,220],"performed":[186],"detection,":[188],"an$F_{1}$of":[189],"25%":[190],"achieved,":[192],"spotting":[200],"ranking":[201,250],"pass":[202],"performed,":[205],"localisation$P{@}10$of":[207],"32%":[208],"obtained.":[210],"While":[211],"these":[212,226,241],"modest":[215],"compared":[216],"idealised":[219],"unordered":[222],"bag-of-word-supervision":[223],"(from":[224],"transcriptions),":[225],"do":[229],"receive":[231],"supervision.":[236],"Further":[237],"analyses":[238],"show":[239,269],"limited":[244],"by":[245],"detection":[248],"pass.":[251],"Moreover,":[252],"individual":[253],"performance":[256,262],"correlated":[258],"tagging":[261],"classifier.":[266],"also":[268],"qualitatively":[270],"how":[271],"semantic":[274],"mistakes":[275],"occur,":[276],"e.g.":[277],"locatessurferwhen":[281],"queried":[282],"withocean.":[283]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":2}],"updated_date":"2026-08-28T12:50:07.497085","created_date":"2025-10-10T00:00:00"}