{"id":"https://openalex.org/W4387623882","doi":"https://doi.org/10.1109/tmm.2023.3323876","title":"Deep Cross-Modal Retrieval Between Spatial Image and Acoustic Speech","display_name":"Deep Cross-Modal Retrieval Between Spatial Image and Acoustic Speech","publication_year":2023,"publication_date":"2023-10-13","ids":{"openalex":"https://openalex.org/W4387623882","doi":"https://doi.org/10.1109/tmm.2023.3323876"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2023.3323876","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3323876","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056495776","display_name":"Xinyuan Qian","orcid":"https://orcid.org/0000-0002-9511-6713"},"institutions":[{"id":"https://openalex.org/I92403157","display_name":"University of Science and Technology Beijing","ror":"https://ror.org/02egmk993","country_code":"CN","type":"education","lineage":["https://openalex.org/I92403157"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinyuan Qian","raw_affiliation_strings":["School of Computer and Communication Engineering, University of Science and Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9511-6713","affiliations":[{"raw_affiliation_string":"School of Computer and Communication Engineering, University of Science and Technology, Beijing, China","institution_ids":["https://openalex.org/I92403157"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100652844","display_name":"Wei Xue","orcid":"https://orcid.org/0000-0002-4942-7748"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Wei Xue","raw_affiliation_strings":["Division of Emerging Interdisciplinary Areas, Hong Kong University of Science and Technology, Hong Kong, SAR, China"],"raw_orcid":"https://orcid.org/0000-0002-4942-7748","affiliations":[{"raw_affiliation_string":"Division of Emerging Interdisciplinary Areas, Hong Kong University of Science and Technology, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042675934","display_name":"Qiquan Zhang","orcid":"https://orcid.org/0000-0001-5089-6317"},"institutions":[{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Qiquan Zhang","raw_affiliation_strings":["School of Electrical Engineering and Telecommunications, University of New South Wales, Sydney, NSW, Australia"],"raw_orcid":"https://orcid.org/0000-0001-5089-6317","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Telecommunications, University of New South Wales, Sydney, NSW, Australia","institution_ids":["https://openalex.org/I31746571"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026034735","display_name":"Ruijie Tao","orcid":"https://orcid.org/0000-0003-0021-5661"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Ruijie Tao","raw_affiliation_strings":["Department of Electrical and Computer Engineering, National University of Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0003-0021-5661","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, National University of Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032690182","display_name":"Haizhou Li","orcid":"https://orcid.org/0000-0001-9158-9401"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]},{"id":"https://openalex.org/I4210099586","display_name":"Shenzhen Research Institute of Big Data","ror":"https://ror.org/00z1gwf89","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210099586"]},{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN","SG"],"is_corresponding":false,"raw_author_name":"Haizhou Li","raw_affiliation_strings":["Shenzhen Research Institute of Big Data, School of Data Science, The Chinese University of Hong Kong, Shenzhen, China","Department of Electrical and Computer Engineering, National University of Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0001-9158-9401","affiliations":[{"raw_affiliation_string":"Shenzhen Research Institute of Big Data, School of Data Science, The Chinese University of Hong Kong, Shenzhen, China","institution_ids":["https://openalex.org/I4210116924","https://openalex.org/I4210099586"]},{"raw_affiliation_string":"Department of Electrical and Computer Engineering, National University of Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.2178,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.80276438,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"26","issue":null,"first_page":"4480","last_page":"4489"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8577466011047363},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5827739834785461},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5422000885009766},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4775353670120239},{"id":"https://openalex.org/keywords/crossmodal","display_name":"Crossmodal","score":0.45892220735549927},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4201755225658417},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.35416147112846375},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3352692723274231},{"id":"https://openalex.org/keywords/visual-perception","display_name":"Visual perception","score":0.09811124205589294}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8577466011047363},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5827739834785461},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5422000885009766},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4775353670120239},{"id":"https://openalex.org/C60115397","wikidata":"https://www.wikidata.org/wiki/Q5188732","display_name":"Crossmodal","level":4,"score":0.45892220735549927},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4201755225658417},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.35416147112846375},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3352692723274231},{"id":"https://openalex.org/C178253425","wikidata":"https://www.wikidata.org/wiki/Q162668","display_name":"Visual perception","level":3,"score":0.09811124205589294},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tmm.2023.3323876","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3323876","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-132160","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-132160","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.5899999737739563,"id":"https://metadata.un.org/sdg/16"}],"awards":[{"id":"https://openalex.org/G2799040894","display_name":null,"funder_award_id":"62271432","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4331663528","display_name":null,"funder_award_id":"62206234","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5088964179","display_name":null,"funder_award_id":"T00120220002","funder_id":"https://openalex.org/F4320331102","funder_display_name":"Shenzhen Research Institute of Big Data"},{"id":"https://openalex.org/G6648122539","display_name":null,"funder_award_id":"62306029","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G861645084","display_name":null,"funder_award_id":"B10120210117-KP02","funder_id":"https://openalex.org/F4320322942","funder_display_name":"Chinese University of Hong Kong"}],"funders":[{"id":"https://openalex.org/F4320320696","display_name":"Agency for Science, Technology and Research","ror":"https://ror.org/036wvzt09"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322942","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48"},{"id":"https://openalex.org/F4320331102","display_name":"Shenzhen Research Institute of Big Data","ror":"https://ror.org/00z1gwf89"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":63,"referenced_works":["https://openalex.org/W1559975096","https://openalex.org/W1586100143","https://openalex.org/W1949478088","https://openalex.org/W2013535308","https://openalex.org/W2024490156","https://openalex.org/W2053667957","https://openalex.org/W2053946370","https://openalex.org/W2105482032","https://openalex.org/W2106277773","https://openalex.org/W2132551672","https://openalex.org/W2145268975","https://openalex.org/W2194775991","https://openalex.org/W2425857666","https://openalex.org/W2546696630","https://openalex.org/W2551572271","https://openalex.org/W2593116425","https://openalex.org/W2606965845","https://openalex.org/W2765440071","https://openalex.org/W2897135094","https://openalex.org/W2936774411","https://openalex.org/W2948037078","https://openalex.org/W2949557830","https://openalex.org/W2949830259","https://openalex.org/W2963055199","https://openalex.org/W2963807156","https://openalex.org/W2964109005","https://openalex.org/W2964339842","https://openalex.org/W2966966150","https://openalex.org/W2967957126","https://openalex.org/W2982573303","https://openalex.org/W3011997138","https://openalex.org/W3038636301","https://openalex.org/W3094550259","https://openalex.org/W3108332675","https://openalex.org/W3121594710","https://openalex.org/W3139813613","https://openalex.org/W3174856432","https://openalex.org/W3176232375","https://openalex.org/W3182683290","https://openalex.org/W3203491020","https://openalex.org/W3213454282","https://openalex.org/W4200528395","https://openalex.org/W4213303451","https://openalex.org/W4214681287","https://openalex.org/W4226351460","https://openalex.org/W4249992252","https://openalex.org/W4282937133","https://openalex.org/W4284898017","https://openalex.org/W4312384316","https://openalex.org/W4312779270","https://openalex.org/W4385245566","https://openalex.org/W4386071707","https://openalex.org/W6606244218","https://openalex.org/W6629823275","https://openalex.org/W6634987543","https://openalex.org/W6679792166","https://openalex.org/W6686207219","https://openalex.org/W6764040762","https://openalex.org/W6784119104","https://openalex.org/W6790019176","https://openalex.org/W6791353385","https://openalex.org/W6802017037","https://openalex.org/W6839307775"],"related_works":["https://openalex.org/W4240440807","https://openalex.org/W953566696","https://openalex.org/W2010220987","https://openalex.org/W2010927954","https://openalex.org/W2085535992","https://openalex.org/W4386123105","https://openalex.org/W3117345873","https://openalex.org/W2933405975","https://openalex.org/W2053951134","https://openalex.org/W2067981595"],"abstract_inverted_index":{"Cross-modal":[0,19],"Retrieval":[1,20,74,113],"(CMR)":[2,21],"is":[3],"formulated":[4],"for":[5],"the":[6,9,26,36,52,62,103,109,123,129,145,149,170],"scenarios":[7],"where":[8],"queries":[10],"and":[11,33,35,57,65,92,136,142,195,202],"retrieval":[12],"results":[13],"are":[14,139],"of":[15,88,180],"different":[16,132],"modalities.":[17],"Existing":[18],"studies":[22],"mainly":[23],"focus":[24],"on":[25],"common":[27],"contextualized":[28],"information":[29,39],"between":[30,55,192],"text":[31],"transcripts":[32],"images,":[34],"synchronized":[37],"event":[38],"in":[40,47,61],"audio-visual":[41],"recordings.":[42],"Unlike":[43],"all":[44],"previous":[45],"works,":[46],"this":[48,77],"article,":[49],"we":[50,79,107,143],"investigate":[51],"geometric":[53],"correspondence":[54],"images":[56,194],"speech":[58,84,98,137,158],"recordings":[59],"captured":[60],"same":[63],"space":[64,162],"formulate":[66],"a":[67,82],"novel":[68,83],"CMR":[69],"task,":[70],"called":[71],"Spatial":[72],"Image-Acoustic":[73],"(SIAR).":[75],"To":[76],"end,":[78],"first":[80],"design":[81],"encoder":[85,159],"that":[86,155],"consists":[87],"convolution":[89],"neural":[90],"networks":[91],"transformer":[93],"layers,":[94],"to":[95,101,121],"learn":[96],"space-aware":[97],"representations.":[99],"Then,":[100],"eliminate":[102],"cross-modal":[104,125],"inherent":[105],"discrepancy,":[106],"propose":[108],"Contrastive":[110],"Speech":[111],"Image":[112],"(CSIR)":[114],"method":[115],"which":[116],"uses":[117],"supervised":[118],"contrastive":[119],"learning":[120],"attract":[122],"same-space":[124],"features":[126,138],"while":[127],"repelling":[128],"ones":[130],"from":[131,163],"spaces.":[133],"Finally,":[134],"image":[135],"directly":[140],"compared":[141],"predict":[144],"SIAR":[146,191],"result":[147],"with":[148,166,198],"maximum":[150],"similarity.":[151],"Extensive":[152],"experiments":[153],"demonstrate":[154],"our":[156,177,184],"proposed":[157],"can":[160,187],"recognize":[161],"human":[164],"speeches":[165,197],"superior":[167],"performance":[168],"over":[169],"other":[171],"prevailing":[172],"networks.":[173],"It":[174],"also":[175],"sets":[176],"penultimate":[178],"goal":[179],"speech-to-speech":[181],"retrieval.":[182],"Furthermore,":[183],"CSIR":[185],"proposal":[186],"successfully":[188],"perform":[189],"bi-directional":[190],"spatial":[193],"reverberant":[196],"promising":[199],"results.":[200],"Code":[201],"data":[203],"will":[204],"be":[205],"available.":[206]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}