{"id":"https://openalex.org/W4407759713","doi":"https://doi.org/10.48550/arxiv.2502.12771","title":"Mind the Gap: Aligning the Brain with Language Models Requires a Nonlinear and Multimodal Approach","display_name":"Mind the Gap: Aligning the Brain with Language Models Requires a Nonlinear and Multimodal Approach","publication_year":2025,"publication_date":"2025-02-18","ids":{"openalex":"https://openalex.org/W4407759713","doi":"https://doi.org/10.48550/arxiv.2502.12771"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.12771","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.12771","pdf_url":"https://arxiv.org/pdf/2502.12771","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.12771","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016422621","display_name":"Danny Dongyeop Han","orcid":"https://orcid.org/0000-0002-9066-9351","observed_orcids":["https://orcid.org/0000-0002-9066-9351"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Danny Dongyeop","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103034087","display_name":"Yunju Cho","orcid":"https://orcid.org/0000-0001-7526-0358","observed_orcids":["https://orcid.org/0000-0001-7526-0358"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cho, Yunju","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033979262","display_name":"Jiook Cha","orcid":"https://orcid.org/0000-0002-5314-7992","observed_orcids":["https://orcid.org/0000-0002-5314-7992"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cha, Jiook","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5111334370","display_name":"Jay-Yoon Lee","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Jay-Yoon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10465","display_name":"Neurobiology of Language and Bilingualism","score":0.9452999830245972,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10465","display_name":"Neurobiology of Language and Bilingualism","score":0.9452999830245972,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10788","display_name":"Neuroscience and Music Perception","score":0.010900000110268593,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.009399999864399433,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/brain-computer-interface","display_name":"brain-computer interface","score":0.4790000021457672},{"id":"https://openalex.org/keywords/speech-perception","display_name":"speech perception","score":0.4729999899864197},{"id":"https://openalex.org/keywords/neural-decoding","display_name":"neural decoding","score":0.5210000276565552},{"id":"https://openalex.org/keywords/nonlinear-forecasting-models","display_name":"nonlinear prediction model","score":0.4390000104904175},{"id":"https://openalex.org/keywords/multimodal-fusion","display_name":"multimodal fusion","score":0.8230000138282776},{"id":"https://openalex.org/keywords/whisper","display_name":"Whisper","score":0.597000002861023},{"id":"https://openalex.org/keywords/neurolinguistics","display_name":"neurolinguistics","score":0.7599999904632568},{"id":"https://openalex.org/keywords/semantic-processing","display_name":"semantic processing","score":0.453000009059906},{"id":"https://openalex.org/keywords/in-silico-testing","display_name":"in silico testing","score":0.5799999833106995},{"id":"https://openalex.org/keywords/natural-speech","display_name":"natural speech","score":0.4699999988079071}],"concepts":[{"id":"https://openalex.org/C158622935","wikidata":"https://www.wikidata.org/wiki/Q660848","display_name":"Nonlinear system","level":2,"score":0.6349592208862305},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5027174949645996},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.46618008613586426},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.35552161931991577},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3537745773792267},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3407366871833801},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.32740217447280884},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.16896754503250122},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.1253097653388977},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.12771","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.12771","pdf_url":"https://arxiv.org/pdf/2502.12771","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.12771","doi":"https://doi.org/10.48550/arxiv.2502.12771","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.12771","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.12771","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.12771","pdf_url":"https://arxiv.org/pdf/2502.12771","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4392903871","https://openalex.org/W4416035042","https://openalex.org/W4402671903","https://openalex.org/W4417256857","https://openalex.org/W7128041283","https://openalex.org/W7166070021","https://openalex.org/W4281765823","https://openalex.org/W4412959787","https://openalex.org/W4308341600","https://openalex.org/W7201975712"],"abstract_inverted_index":{"Self-supervised":[0],"language":[1],"and":[2,31,52,66,73,86,105,114,122,141],"audio":[3,51],"models":[4,14,57],"effectively":[5],"predict":[6],"brain":[7,36,145],"responses":[8],"to":[9,144,153],"speech.":[10],"However,":[11],"traditional":[12,77],"prediction":[13,47,70],"rely":[15],"on":[16],"linear":[17,79],"mappings":[18],"from":[19,55],"unimodal":[20,78],"features,":[21],"despite":[22],"the":[23,135,148],"complex":[24],"integration":[25],"of":[26,139],"auditory":[27,113],"signals":[28],"with":[29,127],"linguistic":[30,53],"semantic":[32,115,124],"information":[33,116],"across":[34],"widespread":[35],"networks":[37],"during":[38],"speech":[39],"comprehension.":[40],"Here,":[41],"we":[42],"introduce":[43],"a":[44,64,84,97],"nonlinear,":[45],"multimodal":[46,142],"model":[48],"that":[49],"combines":[50],"features":[54],"pre-trained":[56],"(e.g.,":[58],"LLAMA,":[59],"Whisper).":[60],"Our":[61],"approach":[62],"achieves":[63],"17.2%":[65],"17.9%":[67],"improvement":[68],"in":[69,119,157],"performance":[71],"(unnormalized":[72],"normalized":[74],"correlation)":[75],"over":[76,90],"models,":[80],"as":[81,83],"well":[82],"7.7%":[85],"14.4%":[87],"improvement,":[88],"respectively,":[89],"prior":[91],"state-of-the-art":[92],"models.":[93],"These":[94],"improvements":[95],"represent":[96],"major":[98],"step":[99],"towards":[100],"future":[101,151],"robust":[102],"in-silico":[103],"testing":[104],"improved":[106],"decoding":[107],"performance.":[108],"They":[109],"also":[110],"reveal":[111],"how":[112],"are":[117],"fused":[118],"motor,":[120],"somatosensory,":[121],"higher-level":[123],"regions,":[125],"aligning":[126],"existing":[128],"neurolinguistic":[129],"theories.":[130],"Overall,":[131],"our":[132],"work":[133],"highlights":[134],"often":[136],"neglected":[137],"potential":[138],"nonlinear":[140],"approaches":[143],"modeling,":[146],"paving":[147],"way":[149],"for":[150],"studies":[152],"embrace":[154],"these":[155],"strategies":[156],"naturalistic":[158],"neurolinguistics":[159],"research.":[160]},"counts_by_year":[],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-02-20T00:00:00"}