{"id":"https://openalex.org/W4385061659","doi":"https://doi.org/10.1109/taffc.2023.3297270","title":"Multi-Rater Consensus Learning for Modeling Multiple Sparse Ratings of Affective Behaviour","display_name":"Multi-Rater Consensus Learning for Modeling Multiple Sparse Ratings of Affective Behaviour","publication_year":2023,"publication_date":"2023-07-20","ids":{"openalex":"https://openalex.org/W4385061659","doi":"https://doi.org/10.1109/taffc.2023.3297270"},"language":"en","primary_location":{"id":"doi:10.1109/taffc.2023.3297270","is_oa":true,"landing_page_url":"https://doi.org/10.1109/taffc.2023.3297270","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1109/taffc.2023.3297270","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083456765","display_name":"Luca Romeo","orcid":"https://orcid.org/0000-0003-1707-0147"},"institutions":[{"id":"https://openalex.org/I30771326","display_name":"Italian Institute of Technology","ror":"https://ror.org/042t93s57","country_code":"IT","type":"facility","lineage":["https://openalex.org/I30771326"]},{"id":"https://openalex.org/I55133342","display_name":"University of Macerata","ror":"https://ror.org/0001fmy77","country_code":"IT","type":"education","lineage":["https://openalex.org/I55133342"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Luca Romeo","raw_affiliation_strings":["Department Economics and Law, University of Macerata, Macerata, Italy","Fondazione Istituto Italiano di Tecnologia, Genova, Italy"],"raw_orcid":"https://orcid.org/0000-0003-1707-0147","affiliations":[{"raw_affiliation_string":"Department Economics and Law, University of Macerata, Macerata, Italy","institution_ids":["https://openalex.org/I55133342"]},{"raw_affiliation_string":"Fondazione Istituto Italiano di Tecnologia, Genova, Italy","institution_ids":["https://openalex.org/I30771326"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005187038","display_name":"Temitayo Olugbade","orcid":"https://orcid.org/0000-0002-2838-6131"},"institutions":[{"id":"https://openalex.org/I162608824","display_name":"University of Sussex","ror":"https://ror.org/00ayhx656","country_code":"GB","type":"education","lineage":["https://openalex.org/I162608824"]},{"id":"https://openalex.org/I4210144746","display_name":"The London College","ror":"https://ror.org/0546ajs61","country_code":"GB","type":"education","lineage":["https://openalex.org/I4210144746"]},{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Temitayo Olugbade","raw_affiliation_strings":["University of Sussex, Brighton, U.K","University College London (UCL), U.K"],"raw_orcid":"https://orcid.org/0000-0002-2838-6131","affiliations":[{"raw_affiliation_string":"University of Sussex, Brighton, U.K","institution_ids":["https://openalex.org/I162608824"]},{"raw_affiliation_string":"University College London (UCL), U.K","institution_ids":["https://openalex.org/I4210144746","https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034260726","display_name":"Massimiliano Pontil","orcid":"https://orcid.org/0000-0001-9415-098X"},"institutions":[{"id":"https://openalex.org/I30771326","display_name":"Italian Institute of Technology","ror":"https://ror.org/042t93s57","country_code":"IT","type":"facility","lineage":["https://openalex.org/I30771326"]},{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB","IT"],"is_corresponding":false,"raw_author_name":"Massimiliano Pontil","raw_affiliation_strings":["Fondazione Istituto Italiano di Tecnologia, Genova, Italy","Department of Computer Science, University College London, U.K"],"raw_orcid":"https://orcid.org/0000-0001-9415-098X","affiliations":[{"raw_affiliation_string":"Fondazione Istituto Italiano di Tecnologia, Genova, Italy","institution_ids":["https://openalex.org/I30771326"]},{"raw_affiliation_string":"Department of Computer Science, University College London, U.K","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5072897829","display_name":"Nadia Bianchi\u2010Berthouze","orcid":"https://orcid.org/0000-0001-8921-0044"},"institutions":[{"id":"https://openalex.org/I4210167018","display_name":"Interaction Design (United Kingdom)","ror":"https://ror.org/03qfyes53","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210167018"]},{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Nadia Bianchi-Berthouze","raw_affiliation_strings":["UCL Interaction Centre, UCL, London, U.K"],"raw_orcid":"https://orcid.org/0000-0001-8921-0044","affiliations":[{"raw_affiliation_string":"UCL Interaction Centre, UCL, London, U.K","institution_ids":["https://openalex.org/I4210167018","https://openalex.org/I45129253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2049,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.53993675,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"15","issue":"3","first_page":"859","last_page":"871"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9930999875068665,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12313","display_name":"Emotions and Moral Behavior","score":0.9704999923706055,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/affect","display_name":"Affect (linguistics)","score":0.6346819400787354},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6171740293502808},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6043850183486938},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5610520243644714},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5136427283287048},{"id":"https://openalex.org/keywords/affective-computing","display_name":"Affective computing","score":0.5085278749465942},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.46145591139793396},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3225468397140503},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.3069738447666168}],"concepts":[{"id":"https://openalex.org/C2776035688","wikidata":"https://www.wikidata.org/wiki/Q1606558","display_name":"Affect (linguistics)","level":2,"score":0.6346819400787354},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6171740293502808},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6043850183486938},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5610520243644714},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5136427283287048},{"id":"https://openalex.org/C6438553","wikidata":"https://www.wikidata.org/wiki/Q1185804","display_name":"Affective computing","level":2,"score":0.5085278749465942},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.46145591139793396},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3225468397140503},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3069738447666168},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/taffc.2023.3297270","is_oa":true,"landing_page_url":"https://doi.org/10.1109/taffc.2023.3297270","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},{"id":"pmh:oai:eprints.ucl.ac.uk.OAI2:10173489","is_oa":true,"landing_page_url":"https://discovery.ucl.ac.uk/id/eprint/10173489/","pdf_url":"https://discovery.ucl.ac.uk/id/eprint/10173489/1/TAC_MRCL_MutliTask-1.pdf","source":{"id":"https://openalex.org/S4306400024","display_name":"UCL Discovery (University College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45129253","host_organization_name":"University College London","host_organization_lineage":["https://openalex.org/I45129253"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Transactions on Affective Computing (2023) (In press).","raw_type":"Article"},{"id":"pmh:oai:u-pad.unimc.it:11393/328390","is_oa":true,"landing_page_url":"https://hdl.handle.net/11393/328390","pdf_url":"https://u-pad.unimc.it/bitstream/11393/328390/1/Multi-Rater_Consensus_Learning_for_Modeling_Multiple_Sparse_Ratings_of_Affective_Behaviour.pdf","source":{"id":"https://openalex.org/S4377196455","display_name":"U-PAD Unimc - Open Digital Publications (University of Macerata)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I55133342","host_organization_name":"University of Macerata","host_organization_lineage":["https://openalex.org/I55133342"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:figshare.com:article/28677497","is_oa":true,"landing_page_url":"https://figshare.com/articles/journal_contribution/Multi-rater_consensus_learning_for_modeling_multiple_sparse_ratings_of_affective_behaviour/28677497","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1109/taffc.2023.3297270","is_oa":true,"landing_page_url":"https://doi.org/10.1109/taffc.2023.3297270","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.5600000023841858}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W67922210","https://openalex.org/W108763474","https://openalex.org/W191327111","https://openalex.org/W1057512088","https://openalex.org/W1918539772","https://openalex.org/W1963596489","https://openalex.org/W1990643818","https://openalex.org/W1996803032","https://openalex.org/W2024221294","https://openalex.org/W2026371421","https://openalex.org/W2035461499","https://openalex.org/W2060719701","https://openalex.org/W2062632672","https://openalex.org/W2064086747","https://openalex.org/W2112458651","https://openalex.org/W2143104527","https://openalex.org/W2159190230","https://openalex.org/W2162825184","https://openalex.org/W2343758848","https://openalex.org/W2473938289","https://openalex.org/W2475282416","https://openalex.org/W2740579382","https://openalex.org/W2765291577","https://openalex.org/W2775288710","https://openalex.org/W2788045938","https://openalex.org/W2793862951","https://openalex.org/W2885050890","https://openalex.org/W2979874173","https://openalex.org/W2979960726","https://openalex.org/W2995556886","https://openalex.org/W3198808971","https://openalex.org/W4280518859","https://openalex.org/W4294410857","https://openalex.org/W6651763732","https://openalex.org/W6679031254","https://openalex.org/W6679959949","https://openalex.org/W6680957539","https://openalex.org/W6682171051","https://openalex.org/W6685936606","https://openalex.org/W6687200099","https://openalex.org/W6748420268","https://openalex.org/W6754778195","https://openalex.org/W6756816618","https://openalex.org/W6800855798","https://openalex.org/W6842580096"],"related_works":["https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2358755282","https://openalex.org/W2625833328","https://openalex.org/W2753175802","https://openalex.org/W2809214204","https://openalex.org/W2341941284"],"abstract_inverted_index":{"The":[0,15,159],"use":[1],"of":[2,35,62,169],"multiple":[3],"raters":[4],"to":[5,19,31,66,120,122],"label":[6,126],"datasets":[7,142,162,197],"is":[8,18,191],"an":[9],"established":[10],"practice":[11],"in":[12,24,51,172],"affective":[13,146],"computing.":[14],"principal":[16],"goal":[17],"reduce":[20],"unwanted":[21],"subjective":[22],"bias":[23],"the":[25,32,42,63,98,118,131,166],"labelling":[26,170],"process.":[27],"Unfortunately,":[28],"this":[29,74,115],"leads":[30],"key":[33],"problem":[34,47],"identifying":[36],"a":[37,52,60,68,78,86,103,109],"ground":[38],"truth":[39],"for":[40,93,130,150,165,182,193],"training":[41],"affect":[43,88,195],"recognition":[44,89,157],"system.":[45],"This":[46],"becomes":[48],"more":[49],"relevant":[50],"sparsely-crossed":[53,199],"annotation":[54],"where":[55],"each":[56,94,124],"rater":[57],"only":[58],"labels":[59],"portion":[61],"full":[64],"dataset":[65],"ensure":[67],"manageable":[69],"workload":[70],"per":[71],"rater.":[72],"In":[73],"paper,":[75],"we":[76],"introduce":[77],"Multi-Rater":[79],"Consensus":[80],"Learning":[81],"(MRCL)":[82],"method":[83],"which":[84],"learns":[85],"representative":[87],"model":[90,119],"that":[91,178,189],"accounts":[92],"rater's":[95,125],"agreement":[96],"with":[97,198],"other":[99],"raters.":[100,134],"MRCL":[101,190],"combines":[102],"multitask":[104],"learning":[105],"(MTL)":[106],"regularizer":[107],"and":[108,154,176],"consensus":[110,132],"loss.":[111],"Unlike":[112],"standard":[113],"MTL,":[114],"approach":[116,138],"allows":[117],"learn":[121],"predict":[123],"while":[127],"explicitly":[128],"accounting":[129],"among":[133],"We":[135],"evaluated":[136],"our":[137,184],"on":[139,144],"two":[140,160],"different":[141,167],"based":[143],"spontaneous":[145],"body":[147],"movement":[148],"expressions":[149],"pain":[151],"behaviour":[152],"detection":[153],"laughter":[155],"type":[156],"respectively.":[158],"naturalistic":[161],"were":[163],"chosen":[164],"forms":[168],"(different":[171],"affect,":[173],"observation":[174],"stimuli,":[175],"raters)":[177],"they":[179],"together":[180],"offer":[181],"evaluating":[183],"approach.":[185],"Empirical":[186],"results":[187],"demonstrate":[188],"effective":[192],"modelling":[194],"from":[196],"multi-rater":[200],"annotation.":[201]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-19T07:52:34.831488","created_date":"2023-07-23T00:00:00"}