{"id":"https://openalex.org/W4407208872","doi":"https://doi.org/10.48550/arxiv.2502.02603","title":"SEAL: Speech Embedding Alignment Learning for Speech Large Language Model with Retrieval-Augmented Generation","display_name":"SEAL: Speech Embedding Alignment Learning for Speech Large Language Model with Retrieval-Augmented Generation","alternate_titles":[],"publication_year":2025,"publication_date":"2025-01-26","ids":{"openalex":"https://openalex.org/W4407208872","doi":"https://doi.org/10.48550/arxiv.2502.02603"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.02603","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.02603","pdf_url":"https://arxiv.org/pdf/2502.02603","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4407208872"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.02603","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102271304","display_name":"Chunyu Sun","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4401727009","display_name":"Sensetime (China)","ror":"https://ror.org/049wsmj07","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401727009"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Sun, Chunyu","raw_affiliation_strings":["SenseTime Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SenseTime Research","institution_ids":["https://openalex.org/I4401727009"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101419905","display_name":"Bingyu Liu","orcid":"https://orcid.org/0000-0003-4817-5355","observed_orcids":["https://orcid.org/0000-0003-4817-5355"]},"institutions":[{"id":"https://openalex.org/I4401727009","display_name":"Sensetime (China)","ror":"https://ror.org/049wsmj07","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401727009"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Liu, Bingyu","raw_affiliation_strings":["SenseTime Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SenseTime Research","institution_ids":["https://openalex.org/I4401727009"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012000087","display_name":"Zhichao Cui","orcid":"https://orcid.org/0000-0003-4326-8309","observed_orcids":["https://orcid.org/0000-0003-4326-8309"]},"institutions":[{"id":"https://openalex.org/I4401727009","display_name":"Sensetime (China)","ror":"https://ror.org/049wsmj07","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401727009"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Cui, Zhichao","raw_affiliation_strings":["SenseTime Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SenseTime Research","institution_ids":["https://openalex.org/I4401727009"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Shi, Junhan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4401727009","display_name":"Sensetime (China)","ror":"https://ror.org/049wsmj07","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401727009"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Shi, Junhan","raw_affiliation_strings":["SenseTime Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SenseTime Research","institution_ids":["https://openalex.org/I4401727009"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038123809","display_name":"Anbin Qi","orcid":"https://orcid.org/0009-0009-1015-7494","observed_orcids":["https://orcid.org/0009-0009-1015-7494"]},"institutions":[{"id":"https://openalex.org/I4401727009","display_name":"Sensetime (China)","ror":"https://ror.org/049wsmj07","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401727009"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Qi, Anbin","raw_affiliation_strings":["SenseTime Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SenseTime Research","institution_ids":["https://openalex.org/I4401727009"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100669567","display_name":"Tianhao Zhang","orcid":"https://orcid.org/0000-0002-6526-7414","observed_orcids":["https://orcid.org/0000-0002-6526-7414"]},"institutions":[{"id":"https://openalex.org/I4401727009","display_name":"Sensetime (China)","ror":"https://ror.org/049wsmj07","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401727009"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Zhang, Tian-hao","raw_affiliation_strings":["SenseTime Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SenseTime Research","institution_ids":["https://openalex.org/I4401727009"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002904368","display_name":"Dinghao Zhou","orcid":"https://orcid.org/0009-0000-8519-4630","observed_orcids":["https://orcid.org/0009-0000-8519-4630"]},"institutions":[{"id":"https://openalex.org/I4401727009","display_name":"Sensetime (China)","ror":"https://ror.org/049wsmj07","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401727009"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Zhou, Dinghao","raw_affiliation_strings":["SenseTime Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SenseTime Research","institution_ids":["https://openalex.org/I4401727009"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5000474748","display_name":"Lewei Lu","orcid":"https://orcid.org/0009-0009-9809-3818","observed_orcids":["https://orcid.org/0009-0009-9809-3818"]},"institutions":[{"id":"https://openalex.org/I4401727009","display_name":"Sensetime (China)","ror":"https://ror.org/049wsmj07","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401727009"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Lu, Lewei","raw_affiliation_strings":["SenseTime Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SenseTime Research","institution_ids":["https://openalex.org/I4401727009"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4401727009"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":17,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9634000062942505,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9634000062942505,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.024900000542402267,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.002400000113993883,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speech-large-language-models","display_name":"speech large language models","score":0.8960000276565552},{"id":"https://openalex.org/keywords/retrieval-augmented-generation","display_name":"retrieval-augmented generation","score":0.9760000109672546},{"id":"https://openalex.org/keywords/embedding-space-alignment","display_name":"embedding alignment","score":0.6660000085830688},{"id":"https://openalex.org/keywords/shared-embedding-space","display_name":"shared embedding space","score":0.6990000009536743},{"id":"https://openalex.org/keywords/speech-encoder","display_name":"speech encoder","score":0.5400000214576721},{"id":"https://openalex.org/keywords/text-encoder","display_name":"text encoder","score":0.8539999723434448},{"id":"https://openalex.org/keywords/multimodal-large-language-models","display_name":"multimodal large language models","score":0.7739999890327454},{"id":"https://openalex.org/keywords/error-propagation","display_name":"error propagation","score":0.5609999895095825}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6788625121116638},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.644039511680603},{"id":"https://openalex.org/C2777755289","wikidata":"https://www.wikidata.org/wiki/Q162919","display_name":"Seal (emblem)","level":2,"score":0.5234957337379456},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5208945274353027},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4766460359096527},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4602550268173218},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4254639744758606},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.10708880424499512},{"id":"https://openalex.org/C195244886","wikidata":"https://www.wikidata.org/wiki/Q41493","display_name":"Ancient history","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.02603","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.02603","pdf_url":"https://arxiv.org/pdf/2502.02603","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.02603","doi":"https://doi.org/10.48550/arxiv.2502.02603","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.02603","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.02603","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.02603","pdf_url":"https://arxiv.org/pdf/2502.02603","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4407208872.pdf","grobid_xml":"https://content.openalex.org/works/W4407208872.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2127141656","https://openalex.org/W2152790380","https://openalex.org/W2160815625","https://openalex.org/W2787948962","https://openalex.org/W2896457183","https://openalex.org/W3135367836","https://openalex.org/W3169320628","https://openalex.org/W4225302959","https://openalex.org/W4226182655","https://openalex.org/W4318718936","https://openalex.org/W4385569624","https://openalex.org/W4385570594","https://openalex.org/W4385573236","https://openalex.org/W4386065803","https://openalex.org/W4386576685","https://openalex.org/W4388017359","https://openalex.org/W4389524500","https://openalex.org/W4392903240","https://openalex.org/W4396913165","https://openalex.org/W4400518537","https://openalex.org/W4400531953","https://openalex.org/W4402672004","https://openalex.org/W4402706274","https://openalex.org/W4403624688","https://openalex.org/W4406461681","https://openalex.org/W7133193597"],"related_works":["https://openalex.org/W7221799977","https://openalex.org/W4414639216","https://openalex.org/W4402683120","https://openalex.org/W4402670461","https://openalex.org/W7140098571","https://openalex.org/W4393028308","https://openalex.org/W4412945327","https://openalex.org/W4408353464","https://openalex.org/W4411120458","https://openalex.org/W7130586922"],"abstract_inverted_index":{"Embedding-based":[0],"retrieval":[1,111,131,165],"models":[2,18,29],"have":[3],"made":[4],"significant":[5],"strides":[6],"in":[7,128,162],"retrieval-augmented":[8],"generation":[9],"(RAG)":[10],"techniques":[11],"for":[12,73,136,158],"text":[13,75,84],"and":[14,56,83,132,152],"multimodal":[15,163],"large":[16],"language":[17,28],"(LLMs)":[19],"applications.":[20],"However,":[21],"when":[22],"it":[23],"comes":[24],"to":[25,35,114],"speech":[26,41,82,130],"larage":[27],"(SLLMs),":[30],"these":[31,61],"methods":[32],"are":[33],"limited":[34],"a":[36,65,88,97,121,159],"two-stage":[37,116],"process,":[38],"where":[39],"automatic":[40],"recognition":[42],"(ASR)":[43],"is":[44],"combined":[45],"with":[46],"text-based":[47],"retrieval.":[48],"This":[49],"sequential":[50],"architecture":[51],"suffers":[52],"from":[53],"high":[54],"latency":[55,105],"error":[57],"propagation.":[58],"To":[59],"address":[60],"limitations,":[62],"we":[63],"propose":[64],"unified":[66],"embedding":[67,99],"framework":[68,79],"that":[69,92],"eliminates":[70],"the":[71,78,125,143,156],"need":[72],"intermediate":[74],"representations.":[76],"Specifically,":[77],"includes":[80],"separate":[81],"encoders,":[85],"followed":[86],"by":[87,106],"shared":[89],"scaling":[90],"layer":[91],"maps":[93],"both":[94],"modalities":[95],"into":[96],"common":[98],"space.":[100],"Our":[101],"model":[102],"reduces":[103],"pipeline":[104],"50\\%":[107],"while":[108],"achieving":[109],"higher":[110],"accuracy":[112],"compared":[113],"traditional":[115],"methods.":[117],"We":[118],"also":[119],"provide":[120],"theoretical":[122],"analysis":[123],"of":[124,145],"challenges":[126],"inherent":[127],"end-to-end":[129],"introduce":[133],"architectural":[134],"principles":[135],"effective":[137],"speech-to-document":[138],"matching.":[139],"Extensive":[140],"experiments":[141],"demonstrate":[142],"robustness":[144],"our":[146],"approach":[147],"across":[148],"diverse":[149],"acoustic":[150],"conditions":[151],"speaker":[153],"variations,":[154],"paving":[155],"way":[157],"new":[160],"paradigm":[161],"SLLMs":[164],"systems.":[166]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4407208872","counts_by_year":[{"year":2026,"cited_by_count":12},{"year":2025,"cited_by_count":5}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2025-02-07T00:00:00"}
