{"id":"https://openalex.org/W6959526928","doi":"https://doi.org/10.1184/r1/26239949.v1","title":"Foundations of Multisensory Artificial Intelligence","display_name":"Foundations of Multisensory Artificial Intelligence","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W6959526928","doi":"https://doi.org/10.1184/r1/26239949.v1"},"language":"en","primary_location":{"id":"pmh:oai:figshare.com:article/26239949","is_oa":true,"landing_page_url":"https://figshare.com/articles/thesis/Foundations_of_Multisensory_Artificial_Intelligence/26239949","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"},"type":"dissertation","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://figshare.com/articles/thesis/Foundations_of_Multisensory_Artificial_Intelligence/26239949","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Liang, Pu","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Liang, Pu","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T13419","display_name":"Ego Development and Educational Practices","score":0.14419999718666077,"subfield":{"id":"https://openalex.org/subfields/3203","display_name":"Clinical Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13419","display_name":"Ego Development and Educational Practices","score":0.14419999718666077,"subfield":{"id":"https://openalex.org/subfields/3203","display_name":"Clinical Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13154","display_name":"Legal and Regulatory Analysis","score":0.14229999482631683,"subfield":{"id":"https://openalex.org/subfields/3313","display_name":"Transportation"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14019","display_name":"Socioeconomic and Demographic Analysis","score":0.048700001090765,"subfield":{"id":"https://openalex.org/subfields/2308","display_name":"Management, Monitoring, Policy and Law"},"field":{"id":"https://openalex.org/fields/23","display_name":"Environmental Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.7021999955177307},{"id":"https://openalex.org/keywords/multimodality","display_name":"Multimodality","score":0.5091999769210815},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.49630001187324524},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.4927999973297119},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"Multimodal learning","score":0.4399999976158142},{"id":"https://openalex.org/keywords/multimodal-interaction","display_name":"Multimodal interaction","score":0.39309999346733093},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.3727000057697296},{"id":"https://openalex.org/keywords/stimulus-modality","display_name":"Stimulus modality","score":0.36980000138282776}],"concepts":[{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.7021999955177307},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6396999955177307},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5309000015258789},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.5091999769210815},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4968000054359436},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.49630001187324524},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.4927999973297119},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.4399999976158142},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.39309999346733093},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.3727000057697296},{"id":"https://openalex.org/C26486553","wikidata":"https://www.wikidata.org/wiki/Q371870","display_name":"Stimulus modality","level":3,"score":0.36980000138282776},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3614000082015991},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.3296000063419342},{"id":"https://openalex.org/C150594956","wikidata":"https://www.wikidata.org/wiki/Q1334829","display_name":"Wearable computer","level":2,"score":0.32910001277923584},{"id":"https://openalex.org/C43540301","wikidata":"https://www.wikidata.org/wiki/Q689971","display_name":"Paradigm shift","level":2,"score":0.2930999994277954},{"id":"https://openalex.org/C2778464652","wikidata":"https://www.wikidata.org/wiki/Q309849","display_name":"Open research","level":2,"score":0.2881999909877777},{"id":"https://openalex.org/C19273510","wikidata":"https://www.wikidata.org/wiki/Q263847","display_name":"Artificial life","level":2,"score":0.2856999933719635},{"id":"https://openalex.org/C2780966255","wikidata":"https://www.wikidata.org/wiki/Q5474306","display_name":"Foundation (evidence)","level":2,"score":0.28529998660087585},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.27090001106262207},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.26649999618530273},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.2522999942302704},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:figshare.com:article/26239949","is_oa":true,"landing_page_url":"https://figshare.com/articles/thesis/Foundations_of_Multisensory_Artificial_Intelligence/26239949","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"},{"id":"doi:10.1184/r1/26239949.v1","is_oa":true,"landing_page_url":"https://doi.org/10.1184/r1/26239949.v1","pdf_url":null,"source":{"id":"https://openalex.org/S7407050927","display_name":"KiltHub Repository","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:figshare.com:article/26239949","is_oa":true,"landing_page_url":"https://figshare.com/articles/thesis/Foundations_of_Multisensory_Artificial_Intelligence/26239949","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"},"sustainable_development_goals":[{"score":0.7292918562889099,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1205139073","display_name":null,"funder_award_id":"1722822","funder_id":"https://openalex.org/F4320332169","funder_display_name":"Directorate for Computer and Information Science and Engineering"},{"id":"https://openalex.org/G6396438598","display_name":null,"funder_award_id":"1750439","funder_id":"https://openalex.org/F4320332169","funder_display_name":"Directorate for Computer and Information Science and Engineering"},{"id":"https://openalex.org/G7274953859","display_name":null,"funder_award_id":"R01MH096951","funder_id":"https://openalex.org/F4320337346","funder_display_name":"National Institute of Mental Health"},{"id":"https://openalex.org/G8360304285","display_name":null,"funder_award_id":"U01MH116923","funder_id":"https://openalex.org/F4320337346","funder_display_name":"National Institute of Mental Health"}],"funders":[{"id":"https://openalex.org/F4320332169","display_name":"Directorate for Computer and Information Science and Engineering","ror":"https://ror.org/025kzpk63"},{"id":"https://openalex.org/F4320337346","display_name":"National Institute of Mental Health","ror":"https://ror.org/04xeg9z08"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Building":[0],"multisensory":[1,196],"artificial":[2,459],"intelligence":[3],"systems":[4],"that":[5,199,337,390],"learn":[6,310],"from":[7,262],"multiple":[8],"sensory":[9,355],"inputs":[10],"such":[11,35,258],"as":[12,36,118,189,191,259],"text,":[13],"speech,":[14],"video,":[15],"real-world":[16,49,286,354,429],"sensors,":[17],"wearable":[18],"devices,":[19],"and":[20,41,47,69,82,107,137,154,158,204,268,314,342,372,385,417,436,456],"medical":[21],"data":[22],"holds":[23],"great":[24],"promise":[25],"for":[26,162,194,234,255,428],"impact":[27,430],"in":[28,37,57,72,111,129,179,206,285,321],"many":[29,202,340,393],"scientific":[30],"areas":[31,374],"with":[32,246,422],"practical":[33,192,279,333],"benefits,":[34],"supporting":[38],"human":[39],"health":[40],"well-being,":[42],"enabling":[43],"multimedia":[44],"content":[45],"processing,":[46],"enhancing":[48],"autonomous":[50],"agents.":[51],"However,":[52],"the":[53,66,73,90,152,168,182,207,211,216,221,263,283,289,325,330,380,405],"breadth":[54],"of":[55,79,92,103,125,167,186,218,224,291,332,369,394,407,415],"progress":[56,173],"multimodal":[58,93,112,130,180,187,219,294,334,386,396,409,458],"research":[59,373],"has":[60,319],"made":[61],"it":[62],"difficult":[63],"to":[64,88,150,201,228,231,249,252,281,299,303,309,353,424],"identify":[65],"common":[67],"themes":[68],"open":[70,160],"questions":[71],"field.":[74],"By":[75],"synthesizing":[76],"a":[77,119,123,235,239,256,293,346,361,366,413],"range":[78,368],"theoretical":[80,240,274],"frameworks":[81],"application":[83],"domains,":[84],"this":[85,146,273,441],"thesis":[86,169,442],"aims":[87],"advance":[89],"foundations":[91,185,217],"machine":[94,183],"learning.":[95],"We":[96,237,357,376,439],"start":[97],"by":[98,443],"defining":[99],"three":[100],"key":[101,177],"principles":[102,117],"modality":[104,302],"heterogeneity,":[105],"connections,":[106],"interactions":[108,284,292,312],"often":[109],"present":[110,238,379],"problems":[113,161],"[371].":[114],"Using":[115,272],"these":[116,311,400,426,450],"foundation,":[120],"we":[121,214,276,328,418],"propose":[122,277],"taxonomy":[124],"six":[126],"core":[127],"challenges":[128],"research:":[131],"representation,":[132],"alignment,":[133],"reasoning,":[134],"generation,":[135],"transference,":[136],"quantification.":[138],"Recent":[139],"technical":[140],"achievements":[141],"will":[142,377],"be":[143],"presented":[144],"through":[145],"taxonomy,":[147],"allowing":[148],"researchers":[149,298],"understand":[151],"similarities":[153],"differences":[155],"across":[156,365,412],"approaches,":[157],"identifying":[159],"future":[163,446],"research.":[164],"The":[165],"bulk":[166],"covers":[170],"our":[171],"recent":[172],"towards":[174],"tackling":[175],"two":[176,278],"prob?lems":[178],"learning:":[181],"learning":[184,322],"interactions,":[188],"well":[190],"methods":[193],"building":[195],"foundation":[197,335,397],"models":[198,336,352,411,427],"generalize":[200,338],"modalities":[203,226,244,341],"tasks":[205],"real":[208],"world.":[209],"In":[210,324],"first":[212,358],"part,":[213,327],"study":[215,329],"interactions:":[220],"basic":[222],"principle":[223],"how":[225,243,445],"combine":[227],"give":[229,250],"rise":[230,251],"new":[232,253],"information":[233,254],"task.":[236],"framework":[241],"formalizing":[242],"interact":[245],"each":[247],"other":[248],"task,":[257],"sarcasm":[260],"identified":[261],"incongruity":[264],"between":[265],"spoken":[266],"words":[267],"vocal":[269],"expressions":[270],"[372].":[271],"framework,":[275],"estimators":[280],"quantify":[282],"datasets.":[287],"Quantifying":[288],"types":[290],"task":[295],"requires":[296],"enables":[297,404],"decide":[300],"which":[301,344],"collect":[304],"[376],":[305],"design":[306,331],"suitable":[307],"approaches":[308],"[374],":[313],"analyze":[315],"whether":[316],"their":[317],"model":[318],"succeeded":[320],"[375].":[323],"second":[326],"over":[339],"tasks,":[343,371,416],"presents":[345],"step":[347],"toward":[348,452],"grounding":[349],"large":[350],"language":[351],"modalities.":[356],"introduce":[359],"MULTIBENCH,":[360],"unified":[362],"large-scale":[363],"benchmark":[364],"wide":[367],"modalities,":[370],"[367].":[375],"also":[378],"cross-modal":[381],"attention":[382],"[101,":[383],"359]":[384],"transformer":[387],"[613]":[388],"architectures":[389,401],"now":[391],"underpin":[392],"today\u2019s":[395],"models.":[398],"Scaling":[399],"on":[402,431],"MULTIBENCH":[403],"creation":[406],"general-purpose":[408],"multitask":[410],"variety":[414],"have":[419],"collaborated":[420],"broadly":[421],"practitioners":[423],"apply":[425],"affective":[432],"computing,":[433],"mental":[434],"health,":[435],"cancer":[437],"prognosis.":[438],"conclude":[440],"discussing":[444],"work":[447],"can":[448],"leverage":[449],"ideas":[451],"more":[453],"general,":[454],"interactive,":[455],"safe":[457],"intelligence.":[460]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}