{"id":"https://openalex.org/W7165625095","doi":"https://doi.org/10.1007/s11263-026-02911-2","title":"Video Object Segmentation-Aware Audio Generation","display_name":"Video Object Segmentation-Aware Audio Generation","publication_year":2026,"publication_date":"2026-06-23","ids":{"openalex":"https://openalex.org/W7165625095","doi":"https://doi.org/10.1007/s11263-026-02911-2"},"language":"en","primary_location":{"id":"doi:10.1007/s11263-026-02911-2","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11263-026-02911-2","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11263-026-02911-2.pdf","source":{"id":"https://openalex.org/S25538012","display_name":"International Journal of Computer Vision","issn_l":"0920-5691","issn":["0920-5691","1573-1405"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Computer Vision","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s11263-026-02911-2.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114412356","display_name":"Ilpo Viertola","orcid":null},"institutions":[{"id":"https://openalex.org/I150589677","display_name":"Tampere University of Applied Sciences","ror":"https://ror.org/00bwtjf83","country_code":"FI","type":"education","lineage":["https://openalex.org/I150589677"]},{"id":"https://openalex.org/I4210133110","display_name":"Tampere University","ror":null,"country_code":"FI","type":null,"lineage":["https://openalex.org/I4210133110"]}],"countries":["FI"],"is_corresponding":true,"raw_author_name":"Ilpo Viertola","raw_affiliation_strings":["Computer Vision Group, Tampere University, Korkeakoulunkatu 7, 33720, Tampere, Finland"],"raw_orcid":"https://orcid.org/0009-0006-2361-8067","affiliations":[{"raw_affiliation_string":"Computer Vision Group, Tampere University, Korkeakoulunkatu 7, 33720, Tampere, Finland","institution_ids":["https://openalex.org/I150589677","https://openalex.org/I4210133110"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008909874","display_name":"Vladimir Iashin","orcid":"https://orcid.org/0000-0001-8879-587X"},"institutions":[{"id":"https://openalex.org/I2802123492","display_name":"Oxford Research Group","ror":"https://ror.org/00z4w4f29","country_code":"GB","type":"nonprofit","lineage":["https://openalex.org/I2802123492"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Vladimir Iashin","raw_affiliation_strings":["Visual Geometry Group, University of Oxford, 25 Banbury Rd, OX2 6NN, Oxford, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Visual Geometry Group, University of Oxford, 25 Banbury Rd, OX2 6NN, Oxford, United Kingdom","institution_ids":["https://openalex.org/I2802123492"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5088180438","display_name":"Esa Rahtu","orcid":"https://orcid.org/0000-0001-8767-0864"},"institutions":[{"id":"https://openalex.org/I150589677","display_name":"Tampere University of Applied Sciences","ror":"https://ror.org/00bwtjf83","country_code":"FI","type":"education","lineage":["https://openalex.org/I150589677"]},{"id":"https://openalex.org/I4210133110","display_name":"Tampere University","ror":null,"country_code":"FI","type":null,"lineage":["https://openalex.org/I4210133110"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Esa Rahtu","raw_affiliation_strings":["Computer Vision Group, Tampere University, Korkeakoulunkatu 7, 33720, Tampere, Finland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Vision Group, Tampere University, Korkeakoulunkatu 7, 33720, Tampere, Finland","institution_ids":["https://openalex.org/I150589677","https://openalex.org/I4210133110"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5114412356"],"corresponding_institution_ids":["https://openalex.org/I150589677","https://openalex.org/I4210133110"],"apc_list":{"value":3690,"currency":"USD","value_usd":3690},"apc_paid":{"value":3690,"currency":"USD","value_usd":3690},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.78024258,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"134","issue":"7","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.616599977016449,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.616599977016449,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.13230000436306,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.10220000147819519,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.6542999744415283},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6051999926567078},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5404000282287598},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4993000030517578},{"id":"https://openalex.org/keywords/object","display_name":"Object (grammar)","score":0.460099995136261},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.4284999966621399},{"id":"https://openalex.org/keywords/video-tracking","display_name":"Video tracking","score":0.40220001339912415},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.3970000147819519},{"id":"https://openalex.org/keywords/audio-signal-processing","display_name":"Audio signal processing","score":0.3817000091075897}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8554999828338623},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.6542999744415283},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6370000243186951},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6051999926567078},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5404000282287598},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.5260999798774719},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4993000030517578},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.460099995136261},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.4284999966621399},{"id":"https://openalex.org/C202474056","wikidata":"https://www.wikidata.org/wiki/Q1931635","display_name":"Video tracking","level":3,"score":0.40220001339912415},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4018000066280365},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.3970000147819519},{"id":"https://openalex.org/C127220857","wikidata":"https://www.wikidata.org/wiki/Q2719318","display_name":"Audio signal processing","level":4,"score":0.3817000091075897},{"id":"https://openalex.org/C124504099","wikidata":"https://www.wikidata.org/wiki/Q56933","display_name":"Image segmentation","level":3,"score":0.3799000084400177},{"id":"https://openalex.org/C125308379","wikidata":"https://www.wikidata.org/wiki/Q363057","display_name":"Market segmentation","level":2,"score":0.3305000066757202},{"id":"https://openalex.org/C160372630","wikidata":"https://www.wikidata.org/wiki/Q4819855","display_name":"Audio analyzer","level":5,"score":0.3125},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3082999885082245},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.29750001430511475},{"id":"https://openalex.org/C65483669","wikidata":"https://www.wikidata.org/wiki/Q3536669","display_name":"Video processing","level":2,"score":0.2791999876499176},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.2777000069618225},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.26820001006126404},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.26080000400543213},{"id":"https://openalex.org/C25694479","wikidata":"https://www.wikidata.org/wiki/Q7446278","display_name":"Segmentation-based object categorization","level":5,"score":0.2565000057220459},{"id":"https://openalex.org/C2776151529","wikidata":"https://www.wikidata.org/wiki/Q3045304","display_name":"Object detection","level":3,"score":0.2565000057220459},{"id":"https://openalex.org/C110446960","wikidata":"https://www.wikidata.org/wiki/Q762316","display_name":"Foley","level":2,"score":0.2558000087738037},{"id":"https://openalex.org/C2780310081","wikidata":"https://www.wikidata.org/wiki/Q1154312","display_name":"Video editing","level":2,"score":0.2556999921798706}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s11263-026-02911-2","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11263-026-02911-2","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11263-026-02911-2.pdf","source":{"id":"https://openalex.org/S25538012","display_name":"International Journal of Computer Vision","issn_l":"0920-5691","issn":["0920-5691","1573-1405"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Computer Vision","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s11263-026-02911-2","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11263-026-02911-2","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11263-026-02911-2.pdf","source":{"id":"https://openalex.org/S25538012","display_name":"International Journal of Computer Vision","issn_l":"0920-5691","issn":["0920-5691","1573-1405"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Computer Vision","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1440761603","display_name":null,"funder_award_id":"353139","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"},{"id":"https://openalex.org/G3904726764","display_name":null,"funder_award_id":"362409","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"},{"id":"https://openalex.org/G6993609826","display_name":"Energy efficient machine perception","funder_award_id":"353139","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"},{"id":"https://openalex.org/G7194505142","display_name":"BERMUDA: Spatial Artificial Intelligence for Visual Geometry and Immersion","funder_award_id":"362409","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"}],"funders":[{"id":"https://openalex.org/F4320321108","display_name":"Academy of Finland","ror":"https://ror.org/05k73zm37"},{"id":"https://openalex.org/F4320322725","display_name":"China Scholarship Council","ror":"https://ror.org/04atp4p48"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7165625095.pdf","grobid_xml":"https://content.openalex.org/works/W7165625095.grobid-xml"},"referenced_works_count":55,"referenced_works":["https://openalex.org/W1907877624","https://openalex.org/W2103869314","https://openalex.org/W2108598243","https://openalex.org/W2593116425","https://openalex.org/W2759171953","https://openalex.org/W2962865004","https://openalex.org/W2981851635","https://openalex.org/W3015371781","https://openalex.org/W3094550259","https://openalex.org/W3115255359","https://openalex.org/W3196974791","https://openalex.org/W3205475937","https://openalex.org/W4288099666","https://openalex.org/W4297676498","https://openalex.org/W4313021454","https://openalex.org/W4313123347","https://openalex.org/W4319050105","https://openalex.org/W4372348103","https://openalex.org/W4386071707","https://openalex.org/W4386075767","https://openalex.org/W4388651136","https://openalex.org/W4390784427","https://openalex.org/W4390872297","https://openalex.org/W4390873054","https://openalex.org/W4390874575","https://openalex.org/W4391272787","https://openalex.org/W4392822068","https://openalex.org/W4392904679","https://openalex.org/W4393160294","https://openalex.org/W4397028833","https://openalex.org/W4398226295","https://openalex.org/W4398796039","https://openalex.org/W4400033239","https://openalex.org/W4400600849","https://openalex.org/W4401307635","https://openalex.org/W4402716264","https://openalex.org/W4402727849","https://openalex.org/W4403580037","https://openalex.org/W4404037650","https://openalex.org/W4404545746","https://openalex.org/W4404612908","https://openalex.org/W4404893154","https://openalex.org/W4408345630","https://openalex.org/W4408352828","https://openalex.org/W4409364205","https://openalex.org/W4411245083","https://openalex.org/W4413144830","https://openalex.org/W4413145987","https://openalex.org/W4414634784","https://openalex.org/W4415795232","https://openalex.org/W4415796144","https://openalex.org/W6967113591","https://openalex.org/W7133196460","https://openalex.org/W7133196924","https://openalex.org/W7133224666"],"related_works":[],"abstract_inverted_index":{"Abstract":[0],"Existing":[1],"multimodal":[2,81],"audio":[3,65,87,114],"generation":[4,88],"models":[5,22],"often":[6],"lack":[7],"precise":[8,32],"user":[9],"control,":[10],"which":[11,67],"limits":[12],"their":[13],"applicability":[14],"in":[15],"professional":[16],"Foley":[17,159],"workflows.":[18],"In":[19],"particular,":[20],"these":[21],"focus":[23],"on":[24,48,72,123],"the":[25,49,58],"entire":[26],"video":[27,62,99],"and":[28,100,109,120,151,166],"do":[29],"not":[30],"provide":[31],"methods":[33,150],"for":[34,89,156,161],"prioritizing":[35],"a":[36,40,79,131,153],"specific":[37],"object":[38,63],"within":[39],"scene,":[41],"generating":[42],"unnecessary":[43],"background":[44],"sounds,":[45],"or":[46],"focusing":[47],"wrong":[50],"objects.":[51],"To":[52,116],"address":[53],"this":[54,118],"gap,":[55],"we":[56,126],"introduce":[57],"novel":[59],"task":[60,119],"of":[61,134],"segmentation-aware":[64,124],"generation,":[66],"explicitly":[68],"conditions":[69],"sound":[70],"synthesis":[71,160],"object-level":[73],"segmentation":[74,95,140],"maps.":[75],"We":[76],"present":[77],"SAGANet,":[78],"new":[80,154],"generative":[82],"model":[83,104],"that":[84],"enables":[85],"controllable":[86],"musical":[90,135,162],"instruments":[91],"by":[92],"leveraging":[93],"visual":[94],"masks":[96],"along":[97],"with":[98,107,139],"textual":[101],"cues.":[102],"Our":[103,142],"provides":[105],"users":[106],"fine-grained":[108],"visually":[110],"localized":[111],"control":[112],"over":[113,147],"generation.":[115],"support":[117],"further":[121],"research":[122],"Foley,":[125],"propose":[127],"Segmented":[128,167],"Music":[129,168],"Solos,":[130],"benchmark":[132],"dataset":[133],"instrument":[136],"performance":[137],"videos":[138],"information.":[141],"method":[143],"demonstrates":[144],"substantial":[145],"improvements":[146],"current":[148],"state-of-the-art":[149],"sets":[152],"standard":[155],"controllable,":[157],"high-fidelity":[158],"audio.":[163],"Code,":[164],"samples,":[165],"Solos":[169],"are":[170],"available":[171],"at":[172],"http://saganet.notion.site":[173],".":[174]},"counts_by_year":[],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2026-06-24T00:00:00"}