{"id":"https://openalex.org/W4381567848","doi":"https://doi.org/10.1109/tnnls.2023.3288022","title":"Visually Guided Sound Source Separation With Audio-Visual Predictive Coding","display_name":"Visually Guided Sound Source Separation With Audio-Visual Predictive Coding","publication_year":2023,"publication_date":"2023-07-12","ids":{"openalex":"https://openalex.org/W4381567848","doi":"https://doi.org/10.1109/tnnls.2023.3288022","pmid":"https://pubmed.ncbi.nlm.nih.gov/37436863"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2023.3288022","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2023.3288022","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2306.10684","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000009166","display_name":"Zengjie Song","orcid":"https://orcid.org/0000-0002-7914-3252"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zengjie Song","raw_affiliation_strings":["School of Mathematics and Statistics, Xi’an Jiaotong University, Xi’an, China"],"raw_orcid":"https://orcid.org/0000-0002-7914-3252","affiliations":[{"raw_affiliation_string":"School of Mathematics and Statistics, Xi’an Jiaotong University, Xi’an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028016065","display_name":"Zhaoxiang Zhang","orcid":"https://orcid.org/0000-0003-2648-3875"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhaoxiang Zhang","raw_affiliation_strings":["Center for Research on Intelligent Perception and Computing, National Laboratory of Pattern Recognition, Institute of Automation, Chinese Academy of Sciences, Beijing, China","National Laboratory of Pattern Recognition, Institute of Automation, Center for Research on Intelligent Perception and Computing, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-2648-3875","affiliations":[{"raw_affiliation_string":"Center for Research on Intelligent Perception and Computing, National Laboratory of Pattern Recognition, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"raw_affiliation_string":"National Laboratory of Pattern Recognition, Institute of Automation, Center for Research on Intelligent Perception and Computing, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2345,"currency":"USD","value_usd":2345},"apc_paid":null,"fwci":0.7753,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.68539461,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"35","issue":"11","first_page":"15528","last_page":"15542"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10283","display_name":"Hearing Loss and Rehabilitation","score":0.9800999760627747,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8268864750862122},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.5549519658088684},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5458080172538757},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.5084304213523865},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4922567903995514},{"id":"https://openalex.org/keywords/audio-visual","display_name":"Audio visual","score":0.46140947937965393},{"id":"https://openalex.org/keywords/audio-signal","display_name":"Audio signal","score":0.4554038941860199},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.39530858397483826},{"id":"https://openalex.org/keywords/speech-coding","display_name":"Speech coding","score":0.26726415753364563},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.08556884527206421}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8268864750862122},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.5549519658088684},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5458080172538757},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.5084304213523865},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4922567903995514},{"id":"https://openalex.org/C3017588708","wikidata":"https://www.wikidata.org/wiki/Q758901","display_name":"Audio visual","level":2,"score":0.46140947937965393},{"id":"https://openalex.org/C64922751","wikidata":"https://www.wikidata.org/wiki/Q4650799","display_name":"Audio signal","level":3,"score":0.4554038941860199},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.39530858397483826},{"id":"https://openalex.org/C13895895","wikidata":"https://www.wikidata.org/wiki/Q3270773","display_name":"Speech coding","level":2,"score":0.26726415753364563},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.08556884527206421},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tnnls.2023.3288022","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2023.3288022","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:37436863","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37436863","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":"Journal Article"},{"id":"pmh:oai:arXiv.org:2306.10684","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.10684","pdf_url":"https://arxiv.org/pdf/2306.10684","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2306.10684","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.10684","pdf_url":"https://arxiv.org/pdf/2306.10684","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3264759890","display_name":null,"funder_award_id":"2021M703489","funder_id":"https://openalex.org/F4320321543","funder_display_name":"China Postdoctoral Science Foundation"},{"id":"https://openalex.org/G3624945331","display_name":null,"funder_award_id":"62006231","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4160143202","display_name":"\u7c7b\u8111\u795e\u7ecf\u7f51\u7edc\u5efa\u6a21\u3001\u5b66\u4e60\u4e0e\u5e94\u7528","funder_award_id":"61836014","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5353429300","display_name":null,"funder_award_id":"U21B2042","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5739981033","display_name":null,"funder_award_id":"61976174.","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7317538059","display_name":null,"funder_award_id":"62072457","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321543","display_name":"China Postdoctoral Science Foundation","ror":"https://ror.org/0426zh255"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4381567848.pdf","grobid_xml":"https://content.openalex.org/works/W4381567848.grobid-xml"},"referenced_works_count":100,"referenced_works":["https://openalex.org/W185399533","https://openalex.org/W1246381107","https://openalex.org/W1580389772","https://openalex.org/W1665214252","https://openalex.org/W1901129140","https://openalex.org/W1974684947","https://openalex.org/W2008817973","https://openalex.org/W2031647436","https://openalex.org/W2092054923","https://openalex.org/W2104298926","https://openalex.org/W2119885245","https://openalex.org/W2127958135","https://openalex.org/W2135029798","https://openalex.org/W2143169494","https://openalex.org/W2150415460","https://openalex.org/W2164899449","https://openalex.org/W2166579744","https://openalex.org/W2187089797","https://openalex.org/W2194775991","https://openalex.org/W2221409856","https://openalex.org/W2286419477","https://openalex.org/W2404400936","https://openalex.org/W2460742184","https://openalex.org/W2511428026","https://openalex.org/W2587994092","https://openalex.org/W2619697695","https://openalex.org/W2759171953","https://openalex.org/W2760103357","https://openalex.org/W2787636496","https://openalex.org/W2796992393","https://openalex.org/W2844030168","https://openalex.org/W2888218464","https://openalex.org/W2890052321","https://openalex.org/W2900292050","https://openalex.org/W2908510526","https://openalex.org/W2952218014","https://openalex.org/W2962699416","https://openalex.org/W2962865004","https://openalex.org/W2962866211","https://openalex.org/W2962949994","https://openalex.org/W2962960500","https://openalex.org/W2962968152","https://openalex.org/W2963218389","https://openalex.org/W2963663420","https://openalex.org/W2963680395","https://openalex.org/W2963807156","https://openalex.org/W2964171275","https://openalex.org/W2964207404","https://openalex.org/W2964229681","https://openalex.org/W2981816492","https://openalex.org/W2981851635","https://openalex.org/W2982619606","https://openalex.org/W2988200020","https://openalex.org/W2990408345","https://openalex.org/W2997240587","https://openalex.org/W3001909981","https://openalex.org/W3002904085","https://openalex.org/W3017343282","https://openalex.org/W3035060554","https://openalex.org/W3037309139","https://openalex.org/W3043840704","https://openalex.org/W3101943858","https://openalex.org/W3108367559","https://openalex.org/W3109606338","https://openalex.org/W3116298410","https://openalex.org/W3123318516","https://openalex.org/W3136499730","https://openalex.org/W3154807520","https://openalex.org/W3159619744","https://openalex.org/W3162322471","https://openalex.org/W3169212847","https://openalex.org/W3170088426","https://openalex.org/W3171007011","https://openalex.org/W4214497471","https://openalex.org/W4221161766","https://openalex.org/W4284898017","https://openalex.org/W4287592659","https://openalex.org/W4289665794","https://openalex.org/W4293665662","https://openalex.org/W4307928257","https://openalex.org/W4307929959","https://openalex.org/W4309971022","https://openalex.org/W4386075686","https://openalex.org/W4386076327","https://openalex.org/W6680012447","https://openalex.org/W6684193366","https://openalex.org/W6729831399","https://openalex.org/W6748775516","https://openalex.org/W6754048563","https://openalex.org/W6757817989","https://openalex.org/W6770805772","https://openalex.org/W6771736971","https://openalex.org/W6772892956","https://openalex.org/W6779411601","https://openalex.org/W6780294235","https://openalex.org/W6790850890","https://openalex.org/W6809915981","https://openalex.org/W6845827088","https://openalex.org/W6846354196","https://openalex.org/W7071105756"],"related_works":["https://openalex.org/W2271369634","https://openalex.org/W3147472394","https://openalex.org/W2047100085","https://openalex.org/W2350550760","https://openalex.org/W578794879","https://openalex.org/W2625296515","https://openalex.org/W3137890128","https://openalex.org/W1984634519","https://openalex.org/W4245955731","https://openalex.org/W2393726419"],"abstract_inverted_index":{"The":[0,103],"framework":[1],"of":[2,10,105,180],"visually":[3],"guided":[4],"sound":[5,20,54,124,137,183],"source":[6],"separation":[7,125,138],"generally":[8],"consists":[9],"three":[11],"parts:":[12],"visual":[13,34,39,117,156],"feature":[14,17,35,46],"extraction,":[15],"multimodal":[16,133],"fusion,":[18,47],"and":[19,41,72,99,119,135,155],"signal":[21],"processing.":[22],"An":[23],"ongoing":[24],"trend":[25],"in":[26,96,140,193],"this":[27,81,94],"field":[28],"has":[29],"been":[30],"to":[31,92,160],"tailor":[32],"involved":[33],"extractor":[36],"for":[37,45,53,114,173],"informative":[38],"guidance":[40],"separately":[42],"devise":[43],"module":[44],"while":[48,198],"utilizing":[49],"U-Net":[50],"by":[51],"default":[52],"analysis.":[55],"However,":[56],"such":[57],"a":[58,84,97,108,120,168],"divide-and-conquer":[59],"paradigm":[60],"is":[61,77,205],"parameter-inefficient":[62],"and,":[63],"meanwhile,":[64],"may":[65],"obtain":[66],"suboptimal":[67],"performance":[68],"as":[69],"jointly":[70],"optimizing":[71],"harmonizing":[73],"various":[74],"model":[75,201],"components":[76],"challengeable.":[78],"By":[79,144],"contrast,":[80],"article":[82],"presents":[83],"novel":[85],"approach,":[86],"dubbed":[87],"audio-visual":[88,178],"predictive":[89,121],"coding":[90,122],"(AVPC),":[91],"tackle":[93],"task":[95],"parameter-efficient":[98],"more":[100],"effective":[101],"manner.":[102],"network":[104,113,126],"AVPC":[106,152,174,189],"features":[107],"simple":[109],"ResNet-based":[110],"video":[111],"analysis":[112],"deriving":[115],"semantic":[116],"features,":[118,131,151],"(PC)-based":[123],"that":[127,188],"can":[128],"extract":[129],"audio":[130,154],"fuse":[132],"information,":[134],"predict":[136],"masks":[139],"the":[141,147,181,200],"same":[142,182],"architecture.":[143],"iteratively":[145],"minimizing":[146],"prediction":[148],"error":[149],"between":[150],"integrates":[153],"information":[157],"recursively,":[158],"leading":[159],"progressively":[161],"improved":[162],"performance.":[163],"In":[164],"addition,":[165],"we":[166],"develop":[167],"valid":[169],"self-supervised":[170],"learning":[171],"strategy":[172],"via":[175],"copredicting":[176],"two":[177],"representations":[179],"source.":[184],"Extensive":[185],"evaluations":[186],"demonstrate":[187],"outperforms":[190],"several":[191],"baselines":[192],"separating":[194],"musical":[195],"instrument":[196],"sounds,":[197],"reducing":[199],"size":[202],"significantly.":[203],"Code":[204],"available":[206],"at:":[207],"https://github.com/zjsong/Audio-Visual-Predictive-Coding.":[208]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1}],"updated_date":"2026-08-28T12:50:07.497085","created_date":"2025-10-10T00:00:00"}