{"id":"https://openalex.org/W4387168955","doi":"https://doi.org/10.3389/fnbot.2023.1281944","title":"Multi-dimensional fusion: transformer and GANs-based multimodal audiovisual perception robot for musical performance art","display_name":"Multi-dimensional fusion: transformer and GANs-based multimodal audiovisual perception robot for musical performance art","publication_year":2023,"publication_date":"2023-09-29","ids":{"openalex":"https://openalex.org/W4387168955","doi":"https://doi.org/10.3389/fnbot.2023.1281944","pmid":"https://pubmed.ncbi.nlm.nih.gov/37841080"},"language":"en","primary_location":{"id":"doi:10.3389/fnbot.2023.1281944","is_oa":true,"landing_page_url":"https://doi.org/10.3389/fnbot.2023.1281944","pdf_url":"https://www.frontiersin.org/articles/10.3389/fnbot.2023.1281944/pdf?isPublishedV2=False","source":{"id":"https://openalex.org/S115606517","display_name":"Frontiers in Neurorobotics","issn_l":"1662-5218","issn":["1662-5218"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320527","host_organization_name":"Frontiers Media","host_organization_lineage":["https://openalex.org/P4310320527"],"host_organization_lineage_names":["Frontiers Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Neurorobotics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.frontiersin.org/articles/10.3389/fnbot.2023.1281944/pdf?isPublishedV2=False","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100736542","display_name":"Shiyi Lu","orcid":"https://orcid.org/0000-0001-8239-1145"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Shiyi Lu","raw_affiliation_strings":["School of Music and Dance, Shanxi Vocational University of Engineering Science and Technology, Jinzhong, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Music and Dance, Shanxi Vocational University of Engineering Science and Technology, Jinzhong, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100408494","display_name":"Panpan Wang","orcid":"https://orcid.org/0000-0003-1387-5140"},"institutions":[{"id":"https://openalex.org/I141103825","display_name":"Nanchang Institute of Technology","ror":"https://ror.org/00avfj807","country_code":"CN","type":"education","lineage":["https://openalex.org/I141103825"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Panpan Wang","raw_affiliation_strings":["School of Computer Information Engineering, Nanchang Institute of Technology, Nanchang, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Information Engineering, Nanchang Institute of Technology, Nanchang, China","institution_ids":["https://openalex.org/I141103825"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5100736542"],"corresponding_institution_ids":[],"apc_list":{"value":2950,"currency":"USD","value_usd":2950},"apc_paid":{"value":2950,"currency":"USD","value_usd":2950},"fwci":0.6864,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.66730616,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"17","issue":null,"first_page":"1281944","last_page":"1281944"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.2761000096797943,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.2761000096797943,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.08860000222921371,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.08150000125169754,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7635319828987122},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.6007120609283447},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.42597806453704834},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.424269437789917},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.40247011184692383}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7635319828987122},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.6007120609283447},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.42597806453704834},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.424269437789917},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.40247011184692383},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.3389/fnbot.2023.1281944","is_oa":true,"landing_page_url":"https://doi.org/10.3389/fnbot.2023.1281944","pdf_url":"https://www.frontiersin.org/articles/10.3389/fnbot.2023.1281944/pdf?isPublishedV2=False","source":{"id":"https://openalex.org/S115606517","display_name":"Frontiers in Neurorobotics","issn_l":"1662-5218","issn":["1662-5218"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320527","host_organization_name":"Frontiers Media","host_organization_lineage":["https://openalex.org/P4310320527"],"host_organization_lineage_names":["Frontiers Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Neurorobotics","raw_type":"journal-article"},{"id":"pmid:37841080","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37841080","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in neurorobotics","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:10570463","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/10570463","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC10570463/pdf/fnbot-17-1281944.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Front Neurorobot","raw_type":"Text"},{"id":"pmh:oai:doaj.org/article:7ed1b9c68b6e491ea494da1549d75c74","is_oa":true,"landing_page_url":"https://doaj.org/article/7ed1b9c68b6e491ea494da1549d75c74","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Frontiers in Neurorobotics, Vol 17 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3389/fnbot.2023.1281944","is_oa":true,"landing_page_url":"https://doi.org/10.3389/fnbot.2023.1281944","pdf_url":"https://www.frontiersin.org/articles/10.3389/fnbot.2023.1281944/pdf?isPublishedV2=False","source":{"id":"https://openalex.org/S115606517","display_name":"Frontiers in Neurorobotics","issn_l":"1662-5218","issn":["1662-5218"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320527","host_organization_name":"Frontiers Media","host_organization_lineage":["https://openalex.org/P4310320527"],"host_organization_lineage_names":["Frontiers Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Neurorobotics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4387168955.pdf"},"referenced_works_count":47,"referenced_works":["https://openalex.org/W41554520","https://openalex.org/W2151392317","https://openalex.org/W2580221632","https://openalex.org/W2759171953","https://openalex.org/W2768282280","https://openalex.org/W2791904197","https://openalex.org/W2887704312","https://openalex.org/W2897963792","https://openalex.org/W2917830802","https://openalex.org/W2950547518","https://openalex.org/W2955202322","https://openalex.org/W2956310820","https://openalex.org/W2963218389","https://openalex.org/W2970842755","https://openalex.org/W2977259558","https://openalex.org/W2987761193","https://openalex.org/W3010049297","https://openalex.org/W3027512603","https://openalex.org/W3041524847","https://openalex.org/W3049228738","https://openalex.org/W3080462905","https://openalex.org/W3093945105","https://openalex.org/W3094467019","https://openalex.org/W3101943858","https://openalex.org/W3111072416","https://openalex.org/W3119219364","https://openalex.org/W3120245141","https://openalex.org/W3128607711","https://openalex.org/W3158446466","https://openalex.org/W3167281848","https://openalex.org/W3193300514","https://openalex.org/W3194723904","https://openalex.org/W4200357302","https://openalex.org/W4206497039","https://openalex.org/W4251857164","https://openalex.org/W4288042648","https://openalex.org/W4293569541","https://openalex.org/W4310844203","https://openalex.org/W4327524109","https://openalex.org/W4367856766","https://openalex.org/W4381250165","https://openalex.org/W6732646663","https://openalex.org/W6746052068","https://openalex.org/W6751104502","https://openalex.org/W6755182157","https://openalex.org/W6840121492","https://openalex.org/W6848441550"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2350741829","https://openalex.org/W2530322880","https://openalex.org/W1596801655","https://openalex.org/W2359140296"],"abstract_inverted_index":{"Introduction:":[0],"In":[1],"the":[2,17,27,124,146,179,190,193,223,226,230,242,284,288,294,298,303,310],"context":[3],"of":[4,19,29,181,187,192,225,232,244,297,305,312],"evolving":[5],"societal":[6],"preferences":[7],"for":[8,320],"deeper":[9],"emotional":[10,40,62,170,253,295],"connections":[11],"in":[12,33,108,145,195,208,219,229,250,270,279],"art,":[13],"this":[14,306],"paper":[15],"explores":[16],"emergence":[18],"multimodal":[20,245],"robot":[21,34,109,134,182,197,233,246],"music":[22,30,139,167,234,247],"performance":[23,209,235,248],"art.":[24,236],"It":[25],"investigates":[26],"fusion":[28],"and":[31,39,50,61,80,94,105,141,155,169,258,265,272],"motion":[32],"performances":[35,74,120],"to":[36,47,55,70,88,102,116,132,213,267,291,308],"enhance":[37],"expressiveness":[38],"impact.":[41,254],"The":[42,97,159,216,238,276],"study":[43,98],"employs":[44],"Transformer":[45,100],"models":[46,101],"combine":[48],"audio":[49,104,257],"video":[51,106],"signals,":[52],"enabling":[53],"robots":[54,261],"better":[56,263],"understand":[57],"music's":[58],"rhythm,":[59],"melody,":[60],"content.":[63],"Generative":[64,111],"Adversarial":[65,112],"Networks":[66,113],"(GANs)":[67],"are":[68,114,143],"utilized":[69],"create":[71],"lifelike":[72],"visual":[73,81,259],"synchronized":[75],"with":[76,123,136,199],"music,":[77,268],"bridging":[78],"auditory":[79],"perception.":[82],"Multimodal":[83,127],"reinforcement":[84,128],"learning":[85,129],"is":[86,130,203],"employed":[87,115],"achieve":[89],"harmonious":[90],"alignment":[91],"between":[92],"sound":[93],"motion.":[95],"Methods:":[96],"leverages":[99],"process":[103],"signals":[107],"performances.":[110,275,323],"generate":[117],"visually":[118],"appealing":[119],"that":[121,283],"align":[122],"musical":[125],"input.":[126],"used":[131],"synchronize":[133],"actions":[135,198],"music.":[137,200,299],"Diverse":[138],"styles":[140,168],"emotions":[142],"considered":[144],"experiments.":[147],"Performance":[148,172],"evaluation":[149],"metrics":[150],"include":[151],"accuracy,":[152],"recall":[153,210,280],"rate,":[154],"F1":[156,220],"score.":[157],"Results:":[158],"proposed":[160,227,285],"approach":[161,228,286,307],"yields":[162],"promising":[163],"results":[164,301],"across":[165],"various":[166],"contexts.":[171],"smoothness":[173],"scores":[174],"exceed":[175],"94":[176],"points,":[177],"demonstrating":[178],"fluidity":[180],"actions.":[183],"An":[184],"accuracy":[185],"rate":[186,211,281],"95%":[188],"highlights":[189],"precision":[191],"system":[194],"aligning":[196],"Notably,":[201],"there":[202],"a":[204],"substantial":[205,277],"33%":[206],"enhancement":[207],"compared":[212],"baseline":[214],"modules.":[215],"collective":[217],"improvement":[218,278],"score":[221],"emphasizes":[222],"advantages":[224],"realm":[231],"Discussion:":[237],"study's":[239],"findings":[240],"demonstrate":[241],"potential":[243,304],"art":[249],"achieving":[251],"heightened":[252],"By":[255],"combining":[256],"cues,":[260],"can":[262],"interpret":[264],"respond":[266],"resulting":[269],"smoother":[271],"more":[273],"precise":[274],"suggests":[282],"enhances":[287],"robots'":[289],"ability":[290],"accurately":[292],"mirror":[293],"nuances":[296],"These":[300],"signify":[302],"transform":[309],"landscape":[311],"artistic":[313],"expression":[314],"through":[315],"robotics,":[316],"opening":[317],"new":[318],"avenues":[319],"emotionally":[321],"resonant":[322]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3}],"updated_date":"2026-07-17T09:13:05.818461","created_date":"2025-10-10T00:00:00"}