{"id":"https://openalex.org/W4385965619","doi":"https://doi.org/10.48550/arxiv.2308.08143","title":"IIANet: An Intra- and Inter-Modality Attention Network for Audio-Visual Speech Separation","display_name":"IIANet: An Intra- and Inter-Modality Attention Network for Audio-Visual Speech Separation","publication_year":2023,"publication_date":"2023-08-16","ids":{"openalex":"https://openalex.org/W4385965619","doi":"https://doi.org/10.48550/arxiv.2308.08143"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2308.08143","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.08143","pdf_url":"https://arxiv.org/pdf/2308.08143","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2308.08143","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100400015","display_name":"Kai Li","orcid":"https://orcid.org/0000-0001-8452-620X","observed_orcids":["https://orcid.org/0000-0001-8452-620X","https://orcid.org/0009-0002-0751-1535"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Kai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006425914","display_name":"Runxuan Yang","orcid":"https://orcid.org/0009-0004-2526-1909","observed_orcids":["https://orcid.org/0009-0004-2526-1909"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Runxuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Sun, Fuchun","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Fuchun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Hu, Xiaolin","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Xiaolin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.986299991607666,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.986299991607666,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.004000000189989805,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12032","display_name":"Multisensory perception and integration","score":0.0024999999441206455,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/audio-visual-speech-separation","display_name":"audio-visual speech separation","score":0.9980000257492065},{"id":"https://openalex.org/keywords/attention-mechanism","display_name":"attention mechanism","score":0.7369999885559082},{"id":"https://openalex.org/keywords/multimodal-fusion","display_name":"multimodal fusion","score":0.7850000262260437},{"id":"https://openalex.org/keywords/timescales","display_name":"time scales","score":0.5},{"id":"https://openalex.org/keywords/modality-specific-features","display_name":"modality-specific features","score":0.7160000205039978},{"id":"https://openalex.org/keywords/inference-efficiency","display_name":"inference efficiency","score":0.5540000200271606}],"concepts":[{"id":"https://openalex.org/C3017588708","wikidata":"https://www.wikidata.org/wiki/Q758901","display_name":"Audio visual","level":2,"score":0.8197934627532959},{"id":"https://openalex.org/C2776061190","wikidata":"https://www.wikidata.org/wiki/Q7451805","display_name":"Separation (statistics)","level":2,"score":0.5954160690307617},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5121808052062988},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.49831223487854004},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.35111960768699646},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.164621502161026},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.07467406988143921}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2308.08143","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.08143","pdf_url":"https://arxiv.org/pdf/2308.08143","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2308.08143","doi":"https://doi.org/10.48550/arxiv.2308.08143","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2308.08143","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2308.08143","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.08143","pdf_url":"https://arxiv.org/pdf/2308.08143","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385965619.pdf","grobid_xml":"https://content.openalex.org/works/W4385965619.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7220464076","https://openalex.org/W7151732418","https://openalex.org/W4402592533","https://openalex.org/W7155042336","https://openalex.org/W4387294212","https://openalex.org/W4283820566","https://openalex.org/W3015445830","https://openalex.org/W4221141685","https://openalex.org/W3143539582","https://openalex.org/W4224920251"],"abstract_inverted_index":{"Recent":[0],"research":[1],"has":[2,167],"made":[3],"significant":[4],"progress":[5],"in":[6,38],"designing":[7],"fusion":[8,20],"modules":[9],"for":[10,64,193],"audio-visual":[11,66,131,138],"speech":[12],"separation.":[13],"However,":[14],"they":[15],"predominantly":[16],"focus":[17],"on":[18,107,135,179],"multi-modal":[19],"at":[21,89,110],"a":[22,50],"single":[23],"temporal":[24,112],"scale":[25],"of":[26,71,74,95,127,148,164,170,190],"auditory":[27],"and":[28,55,79,93,123,143,173,195],"visual":[29],"features":[30,122],"without":[31],"employing":[32],"selective":[33],"attention":[34,62,75,191],"mechanisms,":[35],"which":[36,59],"is":[37,174],"sharp":[39],"contrast":[40],"with":[41],"the":[42,61,84,90,100,117,125,146,161,187],"brain.":[43],"To":[44],"address":[45],"this":[46],"issue,":[47],"We":[48],"propose":[49],"novel":[51],"model":[52],"called":[53],"Intra-":[54],"Inter-Attention":[56],"Network":[57],"(IIANet),":[58],"leverages":[60],"mechanism":[63,192],"efficient":[65,194],"feature":[67],"fusion.":[68,198],"IIANet":[69,165],"consists":[70],"two":[72],"types":[73],"blocks:":[76],"intra-attention":[77],"(IntraA)":[78],"inter-attention":[80],"(InterA)":[81],"blocks,":[82],"where":[83],"InterA":[85],"blocks":[86,115],"are":[87],"distributed":[88],"top,":[91],"middle":[92],"bottom":[94],"IIANet.":[96],"Heavily":[97],"inspired":[98],"by":[99],"way":[101],"how":[102],"human":[103],"brain":[104],"selectively":[105],"focuses":[106],"relevant":[108],"content":[109],"various":[111],"scales,":[113],"these":[114],"maintain":[116],"ability":[118],"to":[119],"learn":[120],"modality-specific":[121],"enable":[124],"extraction":[126],"different":[128],"semantics":[129],"from":[130],"features.":[132],"Comprehensive":[133],"experiments":[134],"three":[136],"standard":[137],"separation":[139,184],"benchmarks":[140],"(LRS2,":[141],"LRS3,":[142],"VoxCeleb2)":[144],"demonstrate":[145],"effectiveness":[147],"IIANet,":[149],"outperforming":[150],"previous":[151],"state-of-the-art":[152],"methods":[153],"while":[154,181],"maintaining":[155],"comparable":[156],"inference":[157],"time.":[158],"In":[159],"particular,":[160],"fast":[162],"version":[163],"(IIANet-fast)":[166],"only":[168],"7%":[169],"CTCNet's":[171],"MACs":[172],"40%":[175],"faster":[176],"than":[177],"CTCNet":[178],"CPUs":[180],"achieving":[182],"better":[183],"quality,":[185],"showing":[186],"great":[188],"potential":[189],"effective":[196],"multimodal":[197]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":2}],"updated_date":"2026-11-12T12:28:57.059296","created_date":"2023-08-18T00:00:00"}