{"id":"https://openalex.org/W7134852894","doi":"https://doi.org/10.48550/arxiv.2603.08126","title":"Foley-Flow: Coordinated Video-to-Audio Generation with Masked Audio-Visual Alignment and Dynamic Conditional Flows","display_name":"Foley-Flow: Coordinated Video-to-Audio Generation with Masked Audio-Visual Alignment and Dynamic Conditional Flows","publication_year":2026,"publication_date":"2026-03-09","ids":{"openalex":"https://openalex.org/W7134852894","doi":"https://doi.org/10.48550/arxiv.2603.08126"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.08126","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.08126","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.08126","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128638876","display_name":"Shentong Mo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mo, Shentong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128650503","display_name":"Yibing Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Yibing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.5302000045776367,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.5302000045776367,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.18279999494552612,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.08789999783039093,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.6575999855995178},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.595300018787384},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.5913000106811523},{"id":"https://openalex.org/keywords/dynamic-range-compression","display_name":"Dynamic range compression","score":0.4537000060081482},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4345000088214874},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.41839998960494995},{"id":"https://openalex.org/keywords/audio-signal","display_name":"Audio signal","score":0.3544999957084656},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.3492000102996826}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7623999714851379},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.6575999855995178},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.595300018787384},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.5913000106811523},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5852000117301941},{"id":"https://openalex.org/C150178126","wikidata":"https://www.wikidata.org/wiki/Q18433212","display_name":"Dynamic range compression","level":2,"score":0.4537000060081482},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4345000088214874},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.41839998960494995},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40880000591278076},{"id":"https://openalex.org/C64922751","wikidata":"https://www.wikidata.org/wiki/Q4650799","display_name":"Audio signal","level":3,"score":0.3544999957084656},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.3492000102996826},{"id":"https://openalex.org/C38349280","wikidata":"https://www.wikidata.org/wiki/Q1434290","display_name":"Flow (mathematics)","level":2,"score":0.3424000144004822},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3375000059604645},{"id":"https://openalex.org/C167310288","wikidata":"https://www.wikidata.org/wiki/Q7564808","display_name":"Sound quality","level":2,"score":0.3319000005722046},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.33180001378059387},{"id":"https://openalex.org/C135343436","wikidata":"https://www.wikidata.org/wiki/Q170406","display_name":"Rhythm","level":2,"score":0.3197000026702881},{"id":"https://openalex.org/C202474056","wikidata":"https://www.wikidata.org/wiki/Q1931635","display_name":"Video tracking","level":3,"score":0.3118000030517578},{"id":"https://openalex.org/C106030495","wikidata":"https://www.wikidata.org/wiki/Q1797012","display_name":"Video compression picture types","level":4,"score":0.30790001153945923},{"id":"https://openalex.org/C127220857","wikidata":"https://www.wikidata.org/wiki/Q2719318","display_name":"Audio signal processing","level":4,"score":0.2937999963760376},{"id":"https://openalex.org/C13895895","wikidata":"https://www.wikidata.org/wiki/Q3270773","display_name":"Speech coding","level":2,"score":0.2865000069141388},{"id":"https://openalex.org/C160372630","wikidata":"https://www.wikidata.org/wiki/Q4819855","display_name":"Audio analyzer","level":5,"score":0.27559998631477356},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.26080000400543213},{"id":"https://openalex.org/C207347870","wikidata":"https://www.wikidata.org/wiki/Q371174","display_name":"Gesture","level":2,"score":0.25679999589920044}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.08126","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.08126","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.08126","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.08126","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Coordinated":[0],"audio":[1,22,55,99,141,167,193,197],"generation":[2,56,149,194],"based":[3],"on":[4,201],"video":[5,30,51,66,109,158,189,234],"inputs":[6],"typically":[7],"requires":[8],"a":[9,35,134],"strict":[10],"audio-visual":[11],"(AV)":[12],"alignment,":[13,183],"where":[14,38,96],"both":[15,61,227],"semantics":[16,74],"and":[17,64,128,177,184,205,229],"rhythmics":[18],"of":[19,106,188],"the":[20,29,39,49,54,97,104,107,113,139,145,161,202],"generated":[21],"segments":[23,100,190],"shall":[24],"correspond":[25],"to":[26,86,164,191,232],"those":[27],"in":[28,70,221],"frames.":[31],"Previous":[32],"studies":[33],"leverage":[34],"two-stage":[36],"design":[37],"AV":[40,73,90,114,182],"encoders":[41,91,115],"are":[42,68,101,117,124,199,226],"firstly":[43],"aligned":[44,125],"via":[45,92],"contrastive":[46,62],"learning,":[47],"then":[48],"encoded":[50],"representations":[52,179],"guide":[53,165,192],"process.":[57],"We":[58],"observe":[59],"that":[60,217,225],"learning":[63],"global":[65],"guidance":[67,105],"effective":[69,220],"aligning":[71],"overall":[72],"while":[75],"limiting":[76],"temporally":[77,156],"rhythmic":[78,129,178],"synchronization.":[79],"In":[80],"this":[81,171,186],"work,":[82],"we":[83,132,173],"propose":[84],"FoleyFlow":[85,218],"first":[87],"align":[88],"unimodal":[89,122],"masked":[93,98,181],"modeling":[94],"training,":[95,112],"recovered":[102],"under":[103,210],"corresponding":[108,166],"segments.":[110],"After":[111],"which":[116],"separately":[118],"pretrained":[119],"using":[120],"only":[121],"data":[123],"with":[126],"semantic":[127,176],"consistency.":[130],"Then,":[131],"develop":[133],"dynamic":[135,152,162],"conditional":[136,153],"flow":[137,148,154],"for":[138],"final":[140],"generation.":[142],"Built":[143],"upon":[144],"efficient":[146],"velocity":[147],"framework,":[150],"our":[151],"utilizes":[155],"varying":[157],"features":[159],"as":[160],"condition":[163],"segment":[168],"generations.":[169],"To":[170],"end,":[172],"extract":[174],"coherent":[175,231],"during":[180],"use":[185],"representation":[187],"temporally.":[195],"Our":[196],"results":[198,209],"evaluated":[200],"standard":[203],"benchmarks":[204],"largely":[206],"surpass":[207],"existing":[208],"several":[211],"metrics.":[212],"The":[213],"superior":[214],"performance":[215],"indicates":[216],"is":[219],"generating":[222],"coordinated":[223],"audios":[224],"semantically":[228],"rhythmically":[230],"various":[233],"sequences.":[235]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-11T00:00:00"}