{"id":"https://openalex.org/W4409727221","doi":"https://doi.org/10.1101/2025.04.15.646250","title":"Comprehensive Neural Representations of Naturalistic Stimuli through Multimodal Deep Learning","display_name":"Comprehensive Neural Representations of Naturalistic Stimuli through Multimodal Deep Learning","publication_year":2025,"publication_date":"2025-04-19","ids":{"openalex":"https://openalex.org/W4409727221","doi":"https://doi.org/10.1101/2025.04.15.646250"},"language":"en","primary_location":{"id":"doi:10.1101/2025.04.15.646250","doi":"https://doi.org/10.1101/2025.04.15.646250","is_oa":true,"landing_page_url":"https://doi.org/10.1101/2025.04.15.646250","pdf_url":"https://www.biorxiv.org/content/biorxiv/early/2025/04/19/2025.04.15.646250.full.pdf","source":{"id":"https://openalex.org/S4306402567","display_name":"bioRxiv (Cold Spring Harbor Laboratory)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2750212522","host_organization_name":"Cold Spring Harbor Laboratory","host_organization_lineage":["https://openalex.org/I2750212522"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"bioRxiv","raw_type":"posted-content"},"volume":null,"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.biorxiv.org/content/biorxiv/early/2025/04/19/2025.04.15.646250.full.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020557227","display_name":"Mingxue Fu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I25254941","display_name":"Beijing Normal University","ror":"https://ror.org/022k4wk35","country_code":"CN","type":"education","lineage":["https://openalex.org/I25254941"]},{"id":"https://openalex.org/I4391768142","display_name":"State Key Laboratory of Cognitive Neuroscience and Learning","ror":"https://ror.org/059y0zb32","country_code":null,"type":"facility","lineage":["https://openalex.org/I25254941","https://openalex.org/I4391768142"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingxue Fu","raw_affiliation_strings":["State Key Laboratory of Cognitive Neuroscience and Learning, and IDG/McGovern Institute for Brain Research, Beijing Normal University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Cognitive Neuroscience and Learning, and IDG/McGovern Institute for Brain Research, Beijing Normal University, Beijing, China","institution_ids":["https://openalex.org/I25254941","https://openalex.org/I4391768142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036612489","display_name":"Guoqiu Chen","orcid":"https://orcid.org/0009-0009-2483-0811","observed_orcids":["https://orcid.org/0009-0009-2483-0811"]},"institutions":[{"id":"https://openalex.org/I25254941","display_name":"Beijing Normal University","ror":"https://ror.org/022k4wk35","country_code":"CN","type":"education","lineage":["https://openalex.org/I25254941"]},{"id":"https://openalex.org/I4391768142","display_name":"State Key Laboratory of Cognitive Neuroscience and Learning","ror":"https://ror.org/059y0zb32","country_code":null,"type":"facility","lineage":["https://openalex.org/I25254941","https://openalex.org/I4391768142"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guoqiu Chen","raw_affiliation_strings":["State Key Laboratory of Cognitive Neuroscience and Learning, and IDG/McGovern Institute for Brain Research, Beijing Normal University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Cognitive Neuroscience and Learning, and IDG/McGovern Institute for Brain Research, Beijing Normal University, Beijing, China","institution_ids":["https://openalex.org/I25254941","https://openalex.org/I4391768142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001300243","display_name":"Yijie Zhang","orcid":"https://orcid.org/0000-0002-6876-9532","observed_orcids":["https://orcid.org/0000-0002-6876-9532","https://orcid.org/0009-0003-6034-0453"]},"institutions":[{"id":"https://openalex.org/I25254941","display_name":"Beijing Normal University","ror":"https://ror.org/022k4wk35","country_code":"CN","type":"education","lineage":["https://openalex.org/I25254941"]},{"id":"https://openalex.org/I4391768142","display_name":"State Key Laboratory of Cognitive Neuroscience and Learning","ror":"https://ror.org/059y0zb32","country_code":null,"type":"facility","lineage":["https://openalex.org/I25254941","https://openalex.org/I4391768142"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yijie Zhang","raw_affiliation_strings":["State Key Laboratory of Cognitive Neuroscience and Learning, and IDG/McGovern Institute for Brain Research, Beijing Normal University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Cognitive Neuroscience and Learning, and IDG/McGovern Institute for Brain Research, Beijing Normal University, Beijing, China","institution_ids":["https://openalex.org/I25254941","https://openalex.org/I4391768142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071665533","display_name":"Mingzhe Zhang","orcid":"https://orcid.org/0009-0004-2758-4146","observed_orcids":["https://orcid.org/0009-0004-2758-4146"]},"institutions":[{"id":"https://openalex.org/I25254941","display_name":"Beijing Normal University","ror":"https://ror.org/022k4wk35","country_code":"CN","type":"education","lineage":["https://openalex.org/I25254941"]},{"id":"https://openalex.org/I4391768142","display_name":"State Key Laboratory of Cognitive Neuroscience and Learning","ror":"https://ror.org/059y0zb32","country_code":null,"type":"facility","lineage":["https://openalex.org/I25254941","https://openalex.org/I4391768142"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingzhe Zhang","raw_affiliation_strings":["State Key Laboratory of Cognitive Neuroscience and Learning, and IDG/McGovern Institute for Brain Research, Beijing Normal University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Cognitive Neuroscience and Learning, and IDG/McGovern Institute for Brain Research, Beijing Normal University, Beijing, China","institution_ids":["https://openalex.org/I25254941","https://openalex.org/I4391768142"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100330971","display_name":"Yin Wang","orcid":"https://orcid.org/0000-0002-3444-000X","observed_orcids":["https://orcid.org/0000-0002-3444-000X"]},"institutions":[{"id":"https://openalex.org/I25254941","display_name":"Beijing Normal University","ror":"https://ror.org/022k4wk35","country_code":"CN","type":"education","lineage":["https://openalex.org/I25254941"]},{"id":"https://openalex.org/I4391768142","display_name":"State Key Laboratory of Cognitive Neuroscience and Learning","ror":"https://ror.org/059y0zb32","country_code":null,"type":"facility","lineage":["https://openalex.org/I25254941","https://openalex.org/I4391768142"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yin Wang","raw_affiliation_strings":["State Key Laboratory of Cognitive Neuroscience and Learning, and IDG/McGovern Institute for Brain Research, Beijing Normal University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-3444-000X","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Cognitive Neuroscience and Learning, and IDG/McGovern Institute for Brain Research, Beijing Normal University, Beijing, China","institution_ids":["https://openalex.org/I25254941","https://openalex.org/I4391768142"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5100330971"],"corresponding_institution_ids":["https://openalex.org/I25254941","https://openalex.org/I4391768142"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5268999934196472,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5268999934196472,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12032","display_name":"Multisensory perception and integration","score":0.21389999985694885,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10465","display_name":"Neurobiology of Language and Bilingualism","score":0.07450000196695328,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/neural-encoding-models","display_name":"neural encoding models","score":0.6790000200271606},{"id":"https://openalex.org/keywords/naturalistic-stimuli","display_name":"naturalistic stimuli","score":0.8489999771118164},{"id":"https://openalex.org/keywords/video-text-alignment","display_name":"video-text alignment","score":0.8889999985694885},{"id":"https://openalex.org/keywords/multimodal-integration","display_name":"multimodal integration","score":0.699999988079071},{"id":"https://openalex.org/keywords/temporal-prediction","display_name":"temporal prediction","score":0.5400000214576721},{"id":"https://openalex.org/keywords/predictive-coding","display_name":"predictive coding","score":0.7260000109672546},{"id":"https://openalex.org/keywords/valor","display_name":"valor","score":0.5619999766349792},{"id":"https://openalex.org/keywords/hierarchical-predictive-coding","display_name":"hierarchical predictive coding","score":0.5630000233650208},{"id":"https://openalex.org/keywords/individual-differences-in-cognitive-ability","display_name":"individual differences in cognitive ability","score":0.6549999713897705}],"concepts":[{"id":"https://openalex.org/C43236755","wikidata":"https://www.wikidata.org/wiki/Q56000","display_name":"Naturalism","level":2,"score":0.6977134943008423},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49999451637268066},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.4996166229248047},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.46694719791412354},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.450131356716156},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.43057122826576233},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.42388418316841125},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.4238263964653015},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.12745457887649536},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.06713631749153137},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1101/2025.04.15.646250","doi":"https://doi.org/10.1101/2025.04.15.646250","is_oa":true,"landing_page_url":"https://doi.org/10.1101/2025.04.15.646250","pdf_url":"https://www.biorxiv.org/content/biorxiv/early/2025/04/19/2025.04.15.646250.full.pdf","source":{"id":"https://openalex.org/S4306402567","display_name":"bioRxiv (Cold Spring Harbor Laboratory)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2750212522","host_organization_name":"Cold Spring Harbor Laboratory","host_organization_lineage":["https://openalex.org/I2750212522"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"bioRxiv","raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.1101/2025.04.15.646250","doi":"https://doi.org/10.1101/2025.04.15.646250","is_oa":true,"landing_page_url":"https://doi.org/10.1101/2025.04.15.646250","pdf_url":"https://www.biorxiv.org/content/biorxiv/early/2025/04/19/2025.04.15.646250.full.pdf","source":{"id":"https://openalex.org/S4306402567","display_name":"bioRxiv (Cold Spring Harbor Laboratory)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2750212522","host_organization_name":"Cold Spring Harbor Laboratory","host_organization_lineage":["https://openalex.org/I2750212522"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"bioRxiv","raw_type":"posted-content"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1790048183","display_name":null,"funder_award_id":"2021ZD0200500","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2960136668","display_name":null,"funder_award_id":"32430041","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3070715974","display_name":null,"funder_award_id":"32422033","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5814787295","display_name":null,"funder_award_id":"2022ZD0211000","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321432","display_name":"Beijing Normal University","ror":"https://ror.org/022k4wk35"},{"id":"https://openalex.org/F4320326840","display_name":"State Key Laboratory of Cognitive Neuroscience and Learning","ror":"https://ror.org/059y0zb32"},{"id":"https://openalex.org/F4320334399","display_name":"McGovern Institute for Brain Research, Massachusetts Institute of Technology","ror":"https://ror.org/05ymca674"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4409727221.pdf","grobid_xml":"https://content.openalex.org/works/W4409727221.grobid-xml"},"referenced_works_count":52,"referenced_works":["https://openalex.org/W1507893557","https://openalex.org/W1967541074","https://openalex.org/W1983208069","https://openalex.org/W2007997142","https://openalex.org/W2011589116","https://openalex.org/W2017040534","https://openalex.org/W2054260111","https://openalex.org/W2063951486","https://openalex.org/W2066269182","https://openalex.org/W2102381086","https://openalex.org/W2125240230","https://openalex.org/W2129409783","https://openalex.org/W2165360032","https://openalex.org/W2171598330","https://openalex.org/W2274405424","https://openalex.org/W2344975321","https://openalex.org/W2400369679","https://openalex.org/W2507706845","https://openalex.org/W2534602978","https://openalex.org/W2537240939","https://openalex.org/W2556947613","https://openalex.org/W2788730280","https://openalex.org/W2896457183","https://openalex.org/W2902687238","https://openalex.org/W2949698470","https://openalex.org/W2951793354","https://openalex.org/W2951924369","https://openalex.org/W2996637160","https://openalex.org/W2997938465","https://openalex.org/W3048423161","https://openalex.org/W3108571598","https://openalex.org/W3135367836","https://openalex.org/W3151549332","https://openalex.org/W3165030117","https://openalex.org/W3186883833","https://openalex.org/W3201124393","https://openalex.org/W3204867366","https://openalex.org/W3211614015","https://openalex.org/W4308294451","https://openalex.org/W4312560592","https://openalex.org/W4316126515","https://openalex.org/W4319462039","https://openalex.org/W4322766928","https://openalex.org/W4367609927","https://openalex.org/W4377866580","https://openalex.org/W4382361744","https://openalex.org/W4386071707","https://openalex.org/W4388625659","https://openalex.org/W4403244151","https://openalex.org/W4403511263","https://openalex.org/W4404724811","https://openalex.org/W4415796580"],"related_works":["https://openalex.org/W4205899436","https://openalex.org/W2462138927","https://openalex.org/W3111427900","https://openalex.org/W4302009627","https://openalex.org/W4212781403","https://openalex.org/W4388989680","https://openalex.org/W2901011898","https://openalex.org/W3160379098","https://openalex.org/W2373442452","https://openalex.org/W2748609568"],"abstract_inverted_index":{"Abstract":[0],"A":[1],"central":[2],"challenge":[3,47],"in":[4,18,34,181],"cognitive":[5,134],"neuroscience":[6],"is":[7,145],"understanding":[8],"how":[9],"the":[10,36,182],"brain":[11,121,151],"represents":[12],"and":[13,63,81,88,179,194],"predicts":[14,56],"complex,":[15],"multimodal":[16,90,143],"experiences":[17],"naturalistic":[19,162,188],"settings.":[20],"Traditional":[21],"neural":[22,58,168],"encoding":[23,53,83],"models,":[24],"often":[25],"based":[26],"on":[27],"unimodal":[28,85],"or":[29],"static":[30,89],"features,":[31],"fall":[32],"short":[33],"capturing":[35],"rich,":[37],"dynamic":[38],"structure":[39],"of":[40,149],"real-world":[41,150,195],"cognition.":[42,196],"Here,":[43],"we":[44,77],"address":[45],"this":[46],"by":[48,60,190],"introducing":[49],"a":[50,69,114,146,171],"video-text":[51],"alignment":[52],"framework":[54,186],"that":[55,130,141,156],"whole-brain":[57],"responses":[59],"integrating":[61],"visual":[62],"linguistic":[64],"features":[65],"across":[66],"time.":[67],"Using":[68],"state-of-the-art":[70],"deep":[71,157],"learning":[72,158],"model":[73,97],"(VALOR;":[74],"Vision-Audio-Language":[75],"Omni-peRception),":[76],"achieve":[78],"more":[79],"accurate":[80],"generalizable":[82],"than":[84],"(AlexNet,":[86],"WordNet)":[87],"(CLIP)":[91],"baselines.":[92],"Beyond":[93],"improving":[94],"prediction,":[95],"our":[96],"automatically":[98],"maps":[99],"cortical":[100],"semantic":[101],"spaces,":[102],"aligning":[103],"with":[104,132,161],"human-annotated":[105],"dimensions":[106],"without":[107],"requiring":[108],"manual":[109],"labeling.":[110],"We":[111],"further":[112],"uncover":[113],"hierarchical":[115],"predictive":[116],"coding":[117],"gradient,":[118],"where":[119],"different":[120],"regions":[122],"anticipate":[123],"future":[124],"events":[125],"over":[126],"distinct":[127],"timescales\u2014an":[128],"organization":[129],"correlates":[131],"individual":[133],"abilities.":[135],"These":[136],"findings":[137],"provide":[138],"new":[139],"evidence":[140],"temporal":[142],"integration":[144],"core":[147],"mechanism":[148],"function.":[152],"Our":[153],"results":[154],"demonstrate":[155],"models":[159],"aligned":[160],"stimuli":[163],"can":[164],"reveal":[165],"ecologically":[166],"valid":[167],"mechanisms,":[169],"offering":[170],"powerful,":[172],"scalable":[173],"approach":[174],"for":[175],"investigating":[176],"perception,":[177],"semantics,":[178],"prediction":[180],"human":[183],"brain.":[184],"This":[185],"advances":[187],"neuroimaging":[189],"bridging":[191],"computational":[192],"modeling":[193]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2025-10-10T00:00:00"}