{"id":"https://openalex.org/W7220806852","doi":null,"title":"Open-vocabulary brain-to-text decoding via cross-modal transfer with large language models","display_name":"Open-vocabulary brain-to-text decoding via cross-modal transfer with large language models","publication_year":2023,"publication_date":"2023-03-13","ids":{"openalex":"https://openalex.org/W7220806852"},"language":"en","primary_location":{"id":"pmh:oai:www.ideals.illinois.edu:2142/120213","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/120213","pdf_url":"https://www.ideals.illinois.edu/items/127434/bitstreams/427012/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"},"volume":null,"type":"dissertation","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.ideals.illinois.edu/items/127434/bitstreams/427012/data.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5157688196","display_name":"Zhenhailong Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]},{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]},{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Wang, Zhenhailong","raw_affiliation_strings":["Computer Science in the Graduate College of the University of Illinois Urbana-Champaign , 2023 Urbana , Illinois","University of North Carolina at Chapel Hill (UNC) ; Professor Shih-Fu Chang from Columbia University ;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science in the Graduate College of the University of Illinois Urbana-Champaign , 2023 Urbana , Illinois","institution_ids":["https://openalex.org/I157725225"]},{"raw_affiliation_string":"University of North Carolina at Chapel Hill (UNC) ; Professor Shih-Fu Chang from Columbia University ;","institution_ids":["https://openalex.org/I114027177","https://openalex.org/I78577930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5157688196"],"corresponding_institution_ids":["https://openalex.org/I114027177","https://openalex.org/I157725225","https://openalex.org/I78577930"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10429","display_name":"EEG and Brain-Computer Interfaces","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10429","display_name":"EEG and Brain-Computer Interfaces","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.0013000000035390258,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0012000000569969416,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cross-modal-transfer","display_name":"cross-modal transfer","score":0.984000027179718},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9559999704360962},{"id":"https://openalex.org/keywords/zero-shot-transfer","display_name":"zero-shot transfer","score":0.722000002861023},{"id":"https://openalex.org/keywords/bart","display_name":"BART","score":0.6240000128746033}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6388000249862671},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.46470001339912415},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4043000042438507},{"id":"https://openalex.org/C2776175482","wikidata":"https://www.wikidata.org/wiki/Q1195816","display_name":"Transfer (computing)","level":2,"score":0.4043000042438507},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3824999928474426},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36090001463890076},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.35010001063346863},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.33239999413490295},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.30390000343322754},{"id":"https://openalex.org/C26022165","wikidata":"https://www.wikidata.org/wiki/Q8091","display_name":"Grammar","level":2,"score":0.28850001096725464},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.28029999136924744},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.26930001378059387},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.26930001378059387},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.2615000009536743}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:www.ideals.illinois.edu:2142/120213","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/120213","pdf_url":"https://www.ideals.illinois.edu/items/127434/bitstreams/427012/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"}],"best_oa_location":{"id":"pmh:oai:www.ideals.illinois.edu:2142/120213","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/120213","pdf_url":"https://www.ideals.illinois.edu/items/127434/bitstreams/427012/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220806852.pdf","grobid_xml":"https://content.openalex.org/works/W7220806852.grobid-xml"},"referenced_works_count":42,"referenced_works":["https://openalex.org/W2133564696","https://openalex.org/W2965373594","https://openalex.org/W2970049541","https://openalex.org/W2981852735","https://openalex.org/W3035448776","https://openalex.org/W3094502228","https://openalex.org/W3126325318","https://openalex.org/W3132730420","https://openalex.org/W3135367836","https://openalex.org/W3168867926","https://openalex.org/W3174770825","https://openalex.org/W3180220247","https://openalex.org/W3195577433","https://openalex.org/W3199693760","https://openalex.org/W3205068155","https://openalex.org/W4221152848","https://openalex.org/W4224308101","https://openalex.org/W4225323055","https://openalex.org/W4225564047","https://openalex.org/W4226278401","https://openalex.org/W4226352076","https://openalex.org/W4229005866","https://openalex.org/W4281633595","https://openalex.org/W4283026156","https://openalex.org/W4283208789","https://openalex.org/W4285069854","https://openalex.org/W4285428875","https://openalex.org/W4290058072","https://openalex.org/W4292945941","https://openalex.org/W4303648971","https://openalex.org/W4304195432","https://openalex.org/W4306820534","https://openalex.org/W4307868887","https://openalex.org/W4311001273","https://openalex.org/W4318718936","https://openalex.org/W4320339901","https://openalex.org/W4385570688","https://openalex.org/W4386065691","https://openalex.org/W7133198447","https://openalex.org/W7133224126","https://openalex.org/W7133231369","https://openalex.org/W7133239427"],"related_works":["https://openalex.org/W7214015142","https://openalex.org/W7034070351","https://openalex.org/W7133220617","https://openalex.org/W7196944556","https://openalex.org/W7197005150","https://openalex.org/W4389519972","https://openalex.org/W6969151813","https://openalex.org/W7220771263","https://openalex.org/W7077075489","https://openalex.org/W4404782586"],"abstract_inverted_index":{"Recent":[0],"research":[1,331],"on":[2,150,289,308],"transformer-based":[3],"large":[4,34,71,194,227,323],"language":[5,19,35,72,85,195,214,228],"models":[6,36,94],"and":[7,21,51,57,211,253,286,335,358],"large-scale":[8,267],"pretraining":[9],"has":[10,250,352,364],"shown":[11],"unprecedented":[12],"success":[13,82],"in":[14,33,83],"the":[15,29,118,129,134,146,151,175,207,212,223,247,258,273,276,298,303,309],"field":[16],"of":[17,24,49,67,117,183,225,241,275],"natural":[18,114],"processing":[20],"beyond.A":[22],"slew":[23],"previous":[25,160],"work":[26],"demonstrated":[27],"that":[28,155,218,246,313],"rich":[30],"knowledge":[31],"embedded":[32],"can":[37,220,317],"be":[38],"effective":[39],"transferred":[40],"across":[41],"different":[42],"modalities,":[43],"to":[44,104,138,200,257,301,340,345,362],"solve":[45],"a":[46,139,180,189,202,226,238,266,281,290,322,359],"wider":[47],"range":[48],"discrinimative":[50],"generative":[52],"tasks,":[53],"e.g.,":[54,230],"vision-language":[55],"understanding":[56],"robotic":[58],"planning.In":[59],"this":[60],"thesis,":[61],"we":[62,132,162,187,219,279],"investigate":[63],"an":[64,355],"uncharted":[65],"area":[66],"cross-modal":[68],"transfer":[69],"with":[70,179,237,255,321,367],"models,":[73],"decoding":[74,84,136,147,235],"brain":[75,88],"signals.State-of-the-art":[76],"brain-to-text":[77,135],"systems":[78],"have":[79],"achieved":[80],"initial":[81],"directly":[86,221],"from":[87,111,123,164],"signals":[89],"using":[90],"modern":[91],"machine":[92],"learning":[93],"such":[95],"as":[96],"recurrent":[97],"neural":[98],"networks.However,":[99],"current":[100],"approaches":[101,120],"are":[102,109],"limited":[103],"small":[105,181],"closed":[106],"vocabularies":[107],"which":[108,168,261],"far":[110],"enough":[112],"for":[113,330],"communication.Additionally,":[115],"most":[116],"high-performing":[119],"require":[121],"data":[122,167,259],"invasive":[124],"devices":[125],"(e.g.,":[126],"ECoG).To":[127],"address":[128],"existing":[130],"limitations,":[131],"extend":[133],"problem":[137],"more":[140,172],"challenging":[141,291],"open":[142],"vocabulary":[143,154,177],"setting,":[144],"where":[145,297],"is":[148,156,169,199,262,300,326],"performed":[149],"entire":[152],"English":[153],"100x":[157],"larger":[158],"than":[159],"work.Furthermore,":[161],"decode":[163],"Electroencephalography":[165],"(EEG)":[166],"non-invasive":[170],"but":[171],"noisy.To":[173],"handle":[174],"huge":[176],"size":[178],"amount":[182],"parallel":[184],"EEG-text":[185],"data,":[186],"propose":[188,280],"novel":[190,282],"model":[191,249],"leveraging":[192],"pretrained":[193],"models.The":[196],"key":[197],"idea":[198],"learn":[201],"good":[203,360],"brainlanguage":[204],"alignment":[205],"between":[206],"\"brain":[208],"embedding":[209,215],"space\"":[210,216],"\"pretrained":[213],"so":[217],"leverage":[222],"power":[224],"model,":[229],"BART.Results":[231],"show":[232,312],"promising":[233],"EEG-to-Text":[234,277],"performance":[236],"BLEU-1":[239],"score":[240],"40.1%.Moreover,":[242],"in-depth":[243],"analysis":[244],"shows":[245],"proposed":[248],"strong":[251],"scalability":[252],"robustness":[254],"respect":[256],"source,":[260],"essential":[263],"towards":[264],"building":[265],"brain-language":[268],"foundation":[269],"model.To":[270],"further":[271],"demonstrate":[272],"effectiveness":[274],"decoder,":[278],"zero-shot":[283,315],"plug-and-play":[284],"framework":[285,316],"evaluate":[287],"it":[288],"downstream":[292],"task,":[293],"EEGbased":[294],"sentiment":[295,305],"classification,":[296],"task":[299],"predict":[302],"sentence's":[304],"based":[306],"solely":[307],"EEG":[310],"signals.We":[311],"our":[314],"outperform":[318],"fullysupervised":[319],"baselines":[320],"margin.The":[324],"code":[325],"made":[327],"publicly":[328],"available":[329],"purpose":[332],"at":[333],"https://github.com/MikeWangWZHL/EEG-To-Text.First":[334],"foremost,":[336],"I":[337],"would":[338],"like":[339],"express":[341],"my":[342,346],"utmost":[343],"gratitude":[344],"advisor,":[347],"Professor":[348],"Heng":[349],"Ji,":[350],"who":[351],"always":[353],"been":[354],"excellent":[356],"mentor":[357],"friend":[361],"me.She":[363],"provided":[365],"me":[366],"endless":[368]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-10-07T00:00:00"}