{"id":"https://openalex.org/W7225752640","doi":"https://doi.org/10.48550/arxiv.2611.12603","title":"Direct Conditioning of Audio Diffusion Transformers on fMRI Reveals Cortical Contributions to Sound Reconstruction","display_name":"Direct Conditioning of Audio Diffusion Transformers on fMRI Reveals Cortical Contributions to Sound Reconstruction","alternate_titles":[],"publication_year":2026,"publication_date":"2026-11-09","ids":{"openalex":"https://openalex.org/W7225752640","doi":"https://doi.org/10.48550/arxiv.2611.12603"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2611.12603","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.12603","pdf_url":"https://arxiv.org/pdf/2611.12603","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W7225752640"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2611.12603","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5099513056","display_name":"Matteo Ciferri","orcid":"https://orcid.org/0009-0006-8484-7533","observed_orcids":["https://orcid.org/0009-0006-8484-7533"]},"institutions":[{"id":"https://openalex.org/I116067653","display_name":"University of Rome Tor Vergata","ror":"https://ror.org/02p77k626","country_code":"IT","type":"education","lineage":["https://openalex.org/I116067653"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Ciferri, Matteo","raw_affiliation_strings":["Department of Biomedicine and Prevention University of Rome Tor Vergata , Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Biomedicine and Prevention University of Rome Tor Vergata , Italy","institution_ids":["https://openalex.org/I116067653"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071652352","display_name":"Tonio Weidler","orcid":"https://orcid.org/0000-0003-1608-5523","observed_orcids":["https://orcid.org/0000-0003-1608-5523"]},"institutions":[{"id":"https://openalex.org/I34352273","display_name":"Maastricht University","ror":"https://ror.org/02jz4aj89","country_code":"NL","type":"education","lineage":["https://openalex.org/I34352273"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Weidler, Tonio","raw_affiliation_strings":["Maastricht Centre for Systems Biology (MaCSBio) Faculty of Science and Engineering Maastricht University , The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Maastricht Centre for Systems Biology (MaCSBio) Faculty of Science and Engineering Maastricht University , The Netherlands","institution_ids":["https://openalex.org/I34352273"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071569531","display_name":"Matteo Ferrante","orcid":"https://orcid.org/0000-0002-3481-0832","observed_orcids":["https://orcid.org/0000-0002-3481-0832"]},"institutions":[{"id":"https://openalex.org/I116067653","display_name":"University of Rome Tor Vergata","ror":"https://ror.org/02p77k626","country_code":"IT","type":"education","lineage":["https://openalex.org/I116067653"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Ferrante, Matteo","raw_affiliation_strings":["Department of Biomedicine and Prevention University of Rome Tor Vergata , Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Biomedicine and Prevention University of Rome Tor Vergata , Italy","institution_ids":["https://openalex.org/I116067653"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010629974","display_name":"Nicola Toschi","orcid":"https://orcid.org/0000-0003-1929-5833","observed_orcids":["https://orcid.org/0000-0003-1929-5833"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Toschi, Nicola","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5000413698","display_name":"Elia Formisano","orcid":"https://orcid.org/0000-0001-5008-2460","observed_orcids":["https://orcid.org/0000-0001-5008-2460"]},"institutions":[{"id":"https://openalex.org/I34352273","display_name":"Maastricht University","ror":"https://ror.org/02jz4aj89","country_code":"NL","type":"education","lineage":["https://openalex.org/I34352273"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Formisano, Elia","raw_affiliation_strings":["Department of Cognitive Neuroscience Faculty of Psychology and Neuroscience Maastricht University , The Netherlands","Maastricht Brain Imaging Center"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Cognitive Neuroscience Faculty of Psychology and Neuroscience Maastricht University , The Netherlands","institution_ids":["https://openalex.org/I34352273"]},{"raw_affiliation_string":"Maastricht Brain Imaging Center","institution_ids":["https://openalex.org/I34352273"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.5547000169754028,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.5547000169754028,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10788","display_name":"Neuroscience and Music Perception","score":0.1185000017285347,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.1143999993801117,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/auditory-cortex","display_name":"auditory cortex","score":0.6549999713897705},{"id":"https://openalex.org/keywords/cross-attention","display_name":"cross-attention","score":0.7229999899864197},{"id":"https://openalex.org/keywords/meuse-river","display_name":"Meuse River","score":0.4189999997615814}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7038999795913696},{"id":"https://openalex.org/C113364801","wikidata":"https://www.wikidata.org/wiki/Q26674","display_name":"High fidelity","level":2,"score":0.5777000188827515},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5289000272750854},{"id":"https://openalex.org/C2779918689","wikidata":"https://www.wikidata.org/wiki/Q3771842","display_name":"Stimulus (psychology)","level":2,"score":0.4726000130176544},{"id":"https://openalex.org/C2780297895","wikidata":"https://www.wikidata.org/wiki/Q18676","display_name":"Auditory cortex","level":2,"score":0.4690000116825104},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.42410001158714294},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.41760000586509705},{"id":"https://openalex.org/C9940772","wikidata":"https://www.wikidata.org/wiki/Q557399","display_name":"Psychoacoustics","level":3,"score":0.40860000252723694},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39010000228881836},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3831999897956848},{"id":"https://openalex.org/C33612138","wikidata":"https://www.wikidata.org/wiki/Q82654","display_name":"Tonotopy","level":3,"score":0.37950000166893005},{"id":"https://openalex.org/C107958047","wikidata":"https://www.wikidata.org/wiki/Q1413664","display_name":"Audio frequency","level":3,"score":0.34360000491142273},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.28700000047683716},{"id":"https://openalex.org/C2779226451","wikidata":"https://www.wikidata.org/wiki/Q903809","display_name":"Functional magnetic resonance imaging","level":2,"score":0.2849999964237213},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.2824999988079071},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.28220000863075256},{"id":"https://openalex.org/C157138929","wikidata":"https://www.wikidata.org/wiki/Q570","display_name":"Loudspeaker","level":2,"score":0.2759000062942505},{"id":"https://openalex.org/C50231774","wikidata":"https://www.wikidata.org/wiki/Q639842","display_name":"Brain mapping","level":2,"score":0.27559998631477356},{"id":"https://openalex.org/C120843803","wikidata":"https://www.wikidata.org/wiki/Q4955807","display_name":"Brain activity and meditation","level":3,"score":0.27549999952316284},{"id":"https://openalex.org/C64922751","wikidata":"https://www.wikidata.org/wiki/Q4650799","display_name":"Audio signal","level":3,"score":0.27469998598098755},{"id":"https://openalex.org/C2777443451","wikidata":"https://www.wikidata.org/wiki/Q821413","display_name":"Auditory system","level":2,"score":0.27469998598098755},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.2694999873638153},{"id":"https://openalex.org/C19118579","wikidata":"https://www.wikidata.org/wiki/Q786423","display_name":"Frequency domain","level":2,"score":0.26589998602867126}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2611.12603","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.12603","pdf_url":"https://arxiv.org/pdf/2611.12603","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2611.12603","doi":"https://doi.org/10.48550/arxiv.2611.12603","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.12603","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2611.12603","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.12603","pdf_url":"https://arxiv.org/pdf/2611.12603","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1843456423","display_name":null,"funder_award_id":"1553 11.10.2022","funder_id":"https://openalex.org/F7311471023","funder_display_name":"NextGenerationEU"},{"id":"https://openalex.org/G2247514905","display_name":null,"funder_award_id":"PE 00000019","funder_id":"https://openalex.org/F7311471023","funder_display_name":"NextGenerationEU"},{"id":"https://openalex.org/G2739164623","display_name":"Aud2Sem: Acoustic to Semantic Transformations in Human Auditory Cortex.","funder_award_id":"406.20.GO.030","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"},{"id":"https://openalex.org/G455327546","display_name":null,"funder_award_id":"PE0000006","funder_id":"https://openalex.org/F7311471023","funder_display_name":"NextGenerationEU"},{"id":"https://openalex.org/G6341469117","display_name":null,"funder_award_id":"11.10.2022","funder_id":"https://openalex.org/F7311471023","funder_display_name":"NextGenerationEU"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"},{"id":"https://openalex.org/F7311471023","display_name":"NextGenerationEU","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7225752640.pdf","grobid_xml":"https://content.openalex.org/works/W7225752640.grobid-xml"},"referenced_works_count":7,"referenced_works":["https://openalex.org/W4310113667","https://openalex.org/W4366208220","https://openalex.org/W4385848774","https://openalex.org/W4386881720","https://openalex.org/W4392972099","https://openalex.org/W6891957894","https://openalex.org/W6948259319"],"related_works":["https://openalex.org/W7117471448","https://openalex.org/W7202184886","https://openalex.org/W7154921569","https://openalex.org/W4412604342","https://openalex.org/W7140720424","https://openalex.org/W7108215582","https://openalex.org/W4416593046","https://openalex.org/W7125895904","https://openalex.org/W4387445259","https://openalex.org/W7155091705"],"abstract_inverted_index":{"Reconstructing":[0],"audio":[1,42,58,145],"from":[2],"brain":[3,32,82],"activity":[4],"remains":[5],"challenging.":[6],"Existing":[7],"brain-to-audio":[8],"pipelines":[9],"typically":[10],"map":[11],"neural":[12,129],"responses":[13,64],"to":[14,41,70,84,134,143],"predefined":[15],"stimulus":[16,74],"representations":[17],"before":[18],"synthesis.":[19],"While":[20],"recent":[21],"work":[22],"in":[23,123],"the":[24,136],"visual":[25],"domain":[26],"has":[27,36],"explored":[28],"direct":[29],"conditioning":[30],"on":[31,61],"activity,":[33],"this":[34],"approach":[35],"not":[37],"yet":[38],"been":[39],"extended":[40],"diffusion":[43,59],"models.":[44],"We":[45],"introduce":[46,118],"MAAS":[47,76],"(Multi-Area":[48],"Auditory":[49],"Synthesis),":[50],"an":[51,72],"end-to-end":[52],"framework":[53],"that":[54],"conditions":[55],"a":[56,68,89,94,119,124],"pretrained":[57],"transformer":[60],"ROI-wise":[62],"fMRI":[63,96],"without":[65],"first":[66],"supervising":[67],"decoder":[69],"predict":[71],"external":[73],"representation.":[75],"uses":[77],"region-specific":[78],"projections":[79],"and":[80,111],"learned":[81],"tokens":[83],"condition":[85],"Stable":[86],"Audio":[87],"through":[88],"dedicated":[90],"cross-attention":[91],"pathway.":[92],"On":[93],"7T":[95],"dataset,":[97],"we":[98,117],"demonstrate":[99],"consistent":[100],"improvements":[101],"over":[102],"baselines,":[103],"including":[104],"two-stage":[105],"fMRI-to-latent":[106],"conditioning,":[107],"across":[108],"both":[109],"acoustic":[110],"semantic":[112],"evaluation":[113],"metrics.":[114],"In":[115],"addition,":[116],"fidelity":[120],"metric":[121],"grounded":[122],"computational":[125],"model":[126],"of":[127,138],"auditory":[128,140],"processing,":[130],"which":[131],"allows":[132],"us":[133],"quantify":[135],"contribution":[137],"individual":[139],"cortical":[141],"regions":[142],"reconstructed":[144],"representations.":[146]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7225752640","counts_by_year":[],"updated_date":"2026-11-15T12:55:33.944570","created_date":"2026-11-11T00:00:00"}
