{"id":"https://openalex.org/W4415065278","doi":"https://doi.org/10.1145/3716553.3750786","title":"Decoding Affective States without Labels: Bimodal Image-brain Supervision","display_name":"Decoding Affective States without Labels: Bimodal Image-brain Supervision","publication_year":2025,"publication_date":"2025-10-11","ids":{"openalex":"https://openalex.org/W4415065278","doi":"https://doi.org/10.1145/3716553.3750786"},"language":"en","primary_location":{"id":"doi:10.1145/3716553.3750786","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3716553.3750786","pdf_url":null,"source":null,"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th International Conference on Multimodal Interaction","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3716553.3750786","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088827750","display_name":"Vadym Gryshchuk","orcid":"https://orcid.org/0000-0002-2839-9514"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Vadym Gryshchuk","raw_affiliation_strings":["University of Copenhagen, Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0000-0002-2839-9514","affiliations":[{"raw_affiliation_string":"University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078639165","display_name":"Maria Maistro","orcid":"https://orcid.org/0000-0002-7001-4817"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Maria Maistro","raw_affiliation_strings":["University of Copenhagen, Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0000-0002-7001-4817","affiliations":[{"raw_affiliation_string":"University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045425016","display_name":"Christina Lioma","orcid":"https://orcid.org/0000-0003-2600-2701"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Christina Lioma","raw_affiliation_strings":["University of Copenhagen, Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0000-0003-2600-2701","affiliations":[{"raw_affiliation_string":"University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064247235","display_name":"Tuukka Ruotsalo","orcid":"https://orcid.org/0000-0002-2203-4928"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]},{"id":"https://openalex.org/I63548447","display_name":"Lappeenranta-Lahti University of Technology","ror":"https://ror.org/0208vgz68","country_code":"FI","type":"education","lineage":["https://openalex.org/I63548447"]}],"countries":["DK","FI"],"is_corresponding":false,"raw_author_name":"Tuukka Ruotsalo","raw_affiliation_strings":["University of Copenhagen, Copenhagen, Denmark and LUT University, Lahti, Finland"],"raw_orcid":"https://orcid.org/0000-0002-2203-4928","affiliations":[{"raw_affiliation_string":"University of Copenhagen, Copenhagen, Denmark and LUT University, Lahti, Finland","institution_ids":["https://openalex.org/I124055696","https://openalex.org/I63548447"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.22845787,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"25","last_page":"34"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9815000295639038,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9815000295639038,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10465","display_name":"Neurobiology of Language and Bilingualism","score":0.970300018787384,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11148","display_name":"Language, Metaphor, and Cognition","score":0.942799985408783,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/valence","display_name":"Valence (chemistry)","score":0.5806000232696533},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.46970000863075256},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.4431999921798706},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.37869998812675476},{"id":"https://openalex.org/keywords/affective-computing","display_name":"Affective computing","score":0.3765000104904175},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.33559998869895935},{"id":"https://openalex.org/keywords/affect","display_name":"Affect (linguistics)","score":0.3301999866962433},{"id":"https://openalex.org/keywords/cognition","display_name":"Cognition","score":0.31529998779296875}],"concepts":[{"id":"https://openalex.org/C168900304","wikidata":"https://www.wikidata.org/wiki/Q171407","display_name":"Valence (chemistry)","level":2,"score":0.5806000232696533},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5738000273704529},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.5248000025749207},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.46970000863075256},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.4431999921798706},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4422000050544739},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.37869998812675476},{"id":"https://openalex.org/C6438553","wikidata":"https://www.wikidata.org/wiki/Q1185804","display_name":"Affective computing","level":2,"score":0.3765000104904175},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34369999170303345},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.33559998869895935},{"id":"https://openalex.org/C2776035688","wikidata":"https://www.wikidata.org/wiki/Q1606558","display_name":"Affect (linguistics)","level":2,"score":0.3301999866962433},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.31529998779296875},{"id":"https://openalex.org/C120843803","wikidata":"https://www.wikidata.org/wiki/Q4955807","display_name":"Brain activity and meditation","level":3,"score":0.3133000135421753},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.29840001463890076},{"id":"https://openalex.org/C2991789358","wikidata":"https://www.wikidata.org/wiki/Q133500","display_name":"Incidental learning","level":2,"score":0.287200003862381},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.27959999442100525},{"id":"https://openalex.org/C178253425","wikidata":"https://www.wikidata.org/wiki/Q162668","display_name":"Visual perception","level":3,"score":0.2687999904155731},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.2671999931335449},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.25450000166893005},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.2535000145435333},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.25099998712539673},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.2500999867916107}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3716553.3750786","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3716553.3750786","pdf_url":null,"source":null,"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th International Conference on Multimodal Interaction","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/629598d6-40f8-41f7-9538-5370463e5f13","is_oa":true,"landing_page_url":"https://researchprofiles.ku.dk/da/publications/629598d6-40f8-41f7-9538-5370463e5f13","pdf_url":"https://curis.ku.dk/ws/files/536149606/3716553.3750786.pdf","source":{"id":"https://openalex.org/S4306401983","display_name":"Research at the University of Copenhagen (University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I124055696","host_organization_name":"University of Copenhagen","host_organization_lineage":["https://openalex.org/I124055696"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"contributionToPeriodical"}],"best_oa_location":{"id":"doi:10.1145/3716553.3750786","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3716553.3750786","pdf_url":null,"source":null,"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th International Conference on Multimodal Interaction","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W1583837637","https://openalex.org/W1980786684","https://openalex.org/W2002055708","https://openalex.org/W2025089570","https://openalex.org/W2064675550","https://openalex.org/W2088580436","https://openalex.org/W2107187638","https://openalex.org/W2108598243","https://openalex.org/W2123229215","https://openalex.org/W2123526233","https://openalex.org/W2147458656","https://openalex.org/W2149628368","https://openalex.org/W2347880541","https://openalex.org/W2421101021","https://openalex.org/W2559463885","https://openalex.org/W2736287575","https://openalex.org/W2787894218","https://openalex.org/W2890089175","https://openalex.org/W2941063500","https://openalex.org/W2941401350","https://openalex.org/W2951583631","https://openalex.org/W2980927909","https://openalex.org/W3041046627","https://openalex.org/W3082414409","https://openalex.org/W3083255269","https://openalex.org/W3093125198","https://openalex.org/W3093531286","https://openalex.org/W3102455230","https://openalex.org/W3129064953","https://openalex.org/W3139408336","https://openalex.org/W3140416091","https://openalex.org/W3155886562","https://openalex.org/W3171007011","https://openalex.org/W3197261948","https://openalex.org/W3199638068","https://openalex.org/W4200630739","https://openalex.org/W4223523352","https://openalex.org/W4224317222","https://openalex.org/W4283819318","https://openalex.org/W4300942166","https://openalex.org/W4308638331","https://openalex.org/W4313256388","https://openalex.org/W4321132568","https://openalex.org/W4372343016","https://openalex.org/W4384642355","https://openalex.org/W4385527149","https://openalex.org/W4386295635","https://openalex.org/W4386634592","https://openalex.org/W4387378260","https://openalex.org/W4387542198","https://openalex.org/W4387665657","https://openalex.org/W4387968305","https://openalex.org/W4388110087","https://openalex.org/W4392125748","https://openalex.org/W4392232009","https://openalex.org/W4399085031","https://openalex.org/W4403791297","https://openalex.org/W4403792182"],"related_works":[],"abstract_inverted_index":{"Identifying":[0],"affective":[1,30,44,77,96,107,188,198,214,234],"responses":[2,137,199],"of":[3,61,134,149,159,164,170,187,192,213],"users":[4],"engaged":[5],"with":[6,224],"digital":[7],"information":[8,12,45],"can":[9,17,236],"provide":[10,230],"valuable":[11],"on":[13,34,183,196],"user":[14,21],"experience":[15],"and":[16,24,74,121,131,167,190,218],"be":[18,237],"used":[19,82],"for":[20,28,210],"modelling,":[22],"retrieval,":[23],"content":[25,47],"recommendation.":[26],"Methods":[27],"recognising":[29],"states":[31,97,189,215,235],"often":[32],"rely":[33],"supervised":[35],"learning":[36,101,212],"from":[37,46,68,239],"a":[38],"single":[39],"modality,":[40],"such":[41],"as":[42],"decoding":[43],"via":[48,98],"computer":[49],"vision,":[50],"human":[51,173],"physiology,":[52],"or":[53],"brain":[54,119,136],"responses.":[55],"These":[56],"approaches":[57],"assume":[58],"the":[59,66,70,75,85,132,135,142,147,197,206],"availability":[60],"paired":[62],"data":[63,117],"that":[64,80,232],"contain":[65],"modality":[67],"which":[69],"prediction":[71],"is":[72,81,245],"made":[73],"corresponding":[76],"state":[78,108],"label":[79],"to":[83,94,123],"supervise":[84],"model.":[86],"Unlike":[87],"previous":[88],"research,":[89],"we":[90],"introduce":[91],"an":[92],"approach":[93,151],"decode":[95],"bimodal":[99,116,211],"contrastive":[100],"without":[102,216],"using":[103,152],"any":[104],"externally":[105],"provided":[106],"labels.":[109],"Our":[110,242],"method,":[111],"entitled":[112],"BALE,":[113],"uses":[114],"only":[115],"(paired":[118],"recordings":[120,158,169],"images)":[122],"self-learn":[124],"latent":[125],"representations":[126,130,133],"by":[127],"contrasting":[128],"image":[129],"evoked":[138],"when":[139,202],"humans":[140],"perceive":[141],"visual":[143,193],"stimuli.":[144],"We":[145,179,204],"demonstrate":[146],"effectiveness":[148],"our":[150,181],"two":[153,184],"publicly":[154],"available":[155],"datasets:":[156],"fNIRS":[157],"images":[160],"showing":[161,175],"various":[162],"levels":[163],"valence":[165],"(positivity/negativity)":[166],"fMRI":[168],"individuals":[171],"viewing":[172],"faces":[174],"different":[176],"emotions":[177],"(joy/anger/contempt/pride/neutral).":[178],"evaluate":[180],"model":[182],"tasks:":[185],"classification":[186],"ranking":[191],"stimuli":[194],"based":[195],"they":[200],"evoke":[201],"perceived.":[203],"report":[205,219],"first":[207],"successful":[208],"results":[209,229],"labels":[217],"performance":[220],"approaching":[221],"models":[222],"trained":[223],"labelled":[225],"supervision":[226],"data.":[227],"The":[228],"evidence":[231],"subjective":[233],"captured":[238],"multimodal":[240],"interactions.":[241],"source":[243],"code":[244],"openly":[246],"released":[247],"at":[248],"https://github.com/VadymV/BALE/.":[249]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-11T00:00:00"}