{"id":"https://openalex.org/W4286359908","doi":"https://doi.org/10.1109/jstsp.2022.3192714","title":"SAMU-XLSR: Semantically-Aligned Multimodal Utterance-Level Cross-Lingual Speech Representation","display_name":"SAMU-XLSR: Semantically-Aligned Multimodal Utterance-Level Cross-Lingual Speech Representation","publication_year":2022,"publication_date":"2022-07-20","ids":{"openalex":"https://openalex.org/W4286359908","doi":"https://doi.org/10.1109/jstsp.2022.3192714"},"language":"en","primary_location":{"id":"doi:10.1109/jstsp.2022.3192714","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2022.3192714","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5035187454","display_name":"Sameer Khurana","orcid":"https://orcid.org/0000-0002-3182-1085"},"institutions":[{"id":"https://openalex.org/I1343180700","display_name":"Intel (United States)","ror":"https://ror.org/01ek73717","country_code":"US","type":"company","lineage":["https://openalex.org/I1343180700"]},{"id":"https://openalex.org/I4210102556","display_name":"K Lab (United States)","ror":"https://ror.org/010zs2155","country_code":"US","type":"company","lineage":["https://openalex.org/I4210102556"]},{"id":"https://openalex.org/I4405258931","display_name":"MIT Computer Science and Artificial Intelligence Laboratory","ror":"https://ror.org/01e85r662","country_code":"US","type":"facility","lineage":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sameer Khurana","raw_affiliation_strings":["MIT Computer Science, Artificial Intelligence Laboratory, Cambridge, MA, USA","MIT CSAIL - MIT Computer Science & Artificial Intelligence Lab (United States)"],"raw_orcid":"https://orcid.org/0000-0002-3182-1085","affiliations":[{"raw_affiliation_string":"MIT Computer Science, Artificial Intelligence Laboratory, Cambridge, MA, USA","institution_ids":[]},{"raw_affiliation_string":"MIT CSAIL - MIT Computer Science & Artificial Intelligence Lab (United States)","institution_ids":["https://openalex.org/I1343180700","https://openalex.org/I4210102556","https://openalex.org/I4405258931"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025212823","display_name":"Antoine Laurent","orcid":"https://orcid.org/0000-0002-2653-1008"},"institutions":[{"id":"https://openalex.org/I4210108471","display_name":"Le Mans Universit\u00e9","ror":"https://ror.org/01mtcc283","country_code":"FR","type":"education","lineage":["https://openalex.org/I4210108471"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Antoine Laurent","raw_affiliation_strings":["LIUM&#x2014;Le Mans University, Le Mans, France"],"raw_orcid":"https://orcid.org/0000-0002-2653-1008","affiliations":[{"raw_affiliation_string":"LIUM&#x2014;Le Mans University, Le Mans, France","institution_ids":["https://openalex.org/I4210108471"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112758056","display_name":"James Glass","orcid":"https://orcid.org/0000-0002-3097-360X"},"institutions":[{"id":"https://openalex.org/I1343180700","display_name":"Intel (United States)","ror":"https://ror.org/01ek73717","country_code":"US","type":"company","lineage":["https://openalex.org/I1343180700"]},{"id":"https://openalex.org/I4210102556","display_name":"K Lab (United States)","ror":"https://ror.org/010zs2155","country_code":"US","type":"company","lineage":["https://openalex.org/I4210102556"]},{"id":"https://openalex.org/I4405258931","display_name":"MIT Computer Science and Artificial Intelligence Laboratory","ror":"https://ror.org/01e85r662","country_code":"US","type":"facility","lineage":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James Glass","raw_affiliation_strings":["MIT Computer Science, Artificial Intelligence Laboratory, Cambridge, MA, USA","MIT CSAIL - MIT Computer Science & Artificial Intelligence Lab (United States)"],"raw_orcid":"https://orcid.org/0000-0002-3097-360X","affiliations":[{"raw_affiliation_string":"MIT Computer Science, Artificial Intelligence Laboratory, Cambridge, MA, USA","institution_ids":[]},{"raw_affiliation_string":"MIT CSAIL - MIT Computer Science & Artificial Intelligence Lab (United States)","institution_ids":["https://openalex.org/I1343180700","https://openalex.org/I4210102556","https://openalex.org/I4405258931"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.8055,"has_fulltext":false,"cited_by_count":30,"citation_normalized_percentile":{"value":0.94114785,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"16","issue":"6","first_page":"1493","last_page":"1504"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/utterance","display_name":"Utterance","score":0.7785358428955078},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.766083836555481},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5899302363395691},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5829436779022217},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.5434255599975586},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.48451846837997437}],"concepts":[{"id":"https://openalex.org/C2775852435","wikidata":"https://www.wikidata.org/wiki/Q258403","display_name":"Utterance","level":2,"score":0.7785358428955078},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.766083836555481},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5899302363395691},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5829436779022217},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5434255599975586},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48451846837997437},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/jstsp.2022.3192714","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2022.3192714","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},{"id":"pmh:oai:HAL:hal-03790203v1","is_oa":false,"landing_page_url":"https://hal.science/hal-03790203","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing, 2022, pp.1-13. &#x27E8;10.1109/JSTSP.2022.3192714&#x27E9;","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.949999988079071}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W2121879602","https://openalex.org/W2127141656","https://openalex.org/W2525778437","https://openalex.org/W2556930864","https://openalex.org/W2612690371","https://openalex.org/W2798389157","https://openalex.org/W2896457183","https://openalex.org/W2921280978","https://openalex.org/W2936774411","https://openalex.org/W2952153923","https://openalex.org/W2962780374","https://openalex.org/W2963672008","https://openalex.org/W2973049979","https://openalex.org/W2973088264","https://openalex.org/W2973157397","https://openalex.org/W2979826702","https://openalex.org/W3001434439","https://openalex.org/W3016011332","https://openalex.org/W3030437843","https://openalex.org/W3032433061","https://openalex.org/W3033038061","https://openalex.org/W3035390927","https://openalex.org/W3039695075","https://openalex.org/W3096918678","https://openalex.org/W3119308075","https://openalex.org/W3152788712","https://openalex.org/W3175301726","https://openalex.org/W3196321886","https://openalex.org/W3197771105","https://openalex.org/W3197828817","https://openalex.org/W3198429080","https://openalex.org/W3198858531","https://openalex.org/W3209059054","https://openalex.org/W3209984917","https://openalex.org/W3213018012","https://openalex.org/W3213029956","https://openalex.org/W4212774754","https://openalex.org/W4221155340","https://openalex.org/W4226033575","https://openalex.org/W6727690538","https://openalex.org/W6729977899","https://openalex.org/W6739901393","https://openalex.org/W6760411109","https://openalex.org/W6767737316","https://openalex.org/W6770596778","https://openalex.org/W6771467084","https://openalex.org/W6773238793","https://openalex.org/W6780218876","https://openalex.org/W6803675045","https://openalex.org/W6810259195","https://openalex.org/W6898634591"],"related_works":["https://openalex.org/W2529301793","https://openalex.org/W2384121599","https://openalex.org/W2038083449","https://openalex.org/W3177678247","https://openalex.org/W1999617572","https://openalex.org/W2944572343","https://openalex.org/W2333799855","https://openalex.org/W2351687372","https://openalex.org/W2004087835","https://openalex.org/W3204019825"],"abstract_inverted_index":{"We":[0,96,208],"propose":[1],"the":[2,58,77,86,112],"(":[3,118],"<inline-formula":[4,106,119,134,143,171,183,197],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[5,11,15,19,23,27,31,35,39,107,120,135,144,172,184,198],"xmlns:xlink=\"http://www.w3.org/1999/xlink\"><tex-math":[6,108,121,136,145,173,185,199],"notation=\"LaTeX\">$\\tt":[7,109,122,137,146,174,186,200],"SAMU\\text{-}XLSR$</tex-math></inline-formula>":[8,138,147,175,201],"):":[9],"<underline":[10,14,18,22,26,30,34,38],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">S</u>":[12,36],"emantically-":[13],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">A</u>":[16],"ligned":[17],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">M</u>":[20],"ultimodal":[21],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">U</u>":[24],"tterance-level":[25],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Cross</u>":[28],"-":[29],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">L</u>":[32],"ingual":[33],"peech":[37],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">R</u>":[40],"epresentation":[41],"learning":[42,70,104],"framework.":[43],"Unlike":[44],"previous":[45],"works":[46],"on":[47,69],"speech":[48,55,74,102,132,152,176,218],"representation":[49,103,163],"learning,":[50],"which":[51],"learns":[52],"multilingual":[53,73,99,131,150],"contextual":[54],"embedding":[56,75,87],"at":[57,76],"resolution":[59,78],"of":[60,79],"an":[61,128],"acoustic":[62,100],"frame":[63],"(10\u201320":[64],"ms),":[65],"this":[66],"work":[67],"focuses":[68],"multimodal":[71,130],"(speech-text)":[72],"a":[80,181],"sentence":[81,189],"(5\u201310":[82],"s)":[83],"such":[84],"that":[85],"vector":[88],"space":[89],"is":[90],"semantically":[91],"aligned":[92],"across":[93,222],"different":[94],"languages.":[95],"combine":[97],"state-of-the-art":[98],"frame-level":[101],"model":[105,125],"XLSR$</tex-math></inline-formula>":[110],"with":[111,148,180],"Language":[113],"Agnostic":[114],"BERT":[115],"Sentence":[116],"Embedding":[117],"LaBSE$</tex-math></inline-formula>":[123,187],")":[124],"to":[126],"create":[127],"utterance-level":[129],"encoder":[133,177,190],".":[139],"Although":[140],"we":[141,169],"train":[142],"only":[149],"transcribed":[151],"data,":[153],"cross-lingual":[154,192,204,215],"speech-text":[155],"and":[156,196,217],"speech-speech":[157],"associations":[158],"emerge":[159],"in":[160,178],"its":[161],"learned":[162],"space.":[164],"To":[165],"substantiate":[166],"our":[167],"claims,":[168],"use":[170],"combination":[179],"pre-trained":[182],"text":[188,216],"for":[191,203],"speech-to-text":[193],"translation":[194,206,219],"retrieval,":[195],"alone":[202],"speech-to-speech":[205],"retrieval.":[207],"highlight":[209],"these":[210],"applications":[211],"by":[212],"performing":[213],"several":[214,223],"retrieval":[220],"tasks":[221],"datasets.":[224]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":15},{"year":2022,"cited_by_count":4}],"updated_date":"2026-07-14T08:27:34.040176","created_date":"2022-07-21T00:00:00"}