{"id":"https://openalex.org/W4390905710","doi":"https://doi.org/10.1109/aciiw59127.2023.10388103","title":"Knowing Your Annotator: Rapidly Testing the Reliability of Affect Annotation","display_name":"Knowing Your Annotator: Rapidly Testing the Reliability of Affect Annotation","publication_year":2023,"publication_date":"2023-09-10","ids":{"openalex":"https://openalex.org/W4390905710","doi":"https://doi.org/10.1109/aciiw59127.2023.10388103"},"language":"en","primary_location":{"id":"doi:10.1109/aciiw59127.2023.10388103","is_oa":false,"landing_page_url":"https://doi.org/10.1109/aciiw59127.2023.10388103","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 11th International Conference on Affective Computing and Intelligent Interaction Workshops and Demos (ACIIW)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044861252","display_name":"Matthew Barthet","orcid":"https://orcid.org/0000-0002-7121-5290"},"institutions":[{"id":"https://openalex.org/I197854408","display_name":"University of Malta","ror":"https://ror.org/03a62bv60","country_code":"MT","type":"education","lineage":["https://openalex.org/I197854408"]}],"countries":["MT"],"is_corresponding":false,"raw_author_name":"Matthew Barthet","raw_affiliation_strings":["Institute of Digital Games, University of Malta,Msida,Malta","Institute of Digital Games, University of Malta, Msida, Malta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Digital Games, University of Malta,Msida,Malta","institution_ids":["https://openalex.org/I197854408"]},{"raw_affiliation_string":"Institute of Digital Games, University of Malta, Msida, Malta","institution_ids":["https://openalex.org/I197854408"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065997245","display_name":"Chintan Trivedi","orcid":"https://orcid.org/0000-0001-5402-5607"},"institutions":[{"id":"https://openalex.org/I197854408","display_name":"University of Malta","ror":"https://ror.org/03a62bv60","country_code":"MT","type":"education","lineage":["https://openalex.org/I197854408"]}],"countries":["MT"],"is_corresponding":false,"raw_author_name":"Chintan Trivedi","raw_affiliation_strings":["Institute of Digital Games, University of Malta,Msida,Malta","Institute of Digital Games, University of Malta, Msida, Malta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Digital Games, University of Malta,Msida,Malta","institution_ids":["https://openalex.org/I197854408"]},{"raw_affiliation_string":"Institute of Digital Games, University of Malta, Msida, Malta","institution_ids":["https://openalex.org/I197854408"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056006892","display_name":"Kosmas Pinitas","orcid":"https://orcid.org/0000-0003-0938-682X"},"institutions":[{"id":"https://openalex.org/I197854408","display_name":"University of Malta","ror":"https://ror.org/03a62bv60","country_code":"MT","type":"education","lineage":["https://openalex.org/I197854408"]}],"countries":["MT"],"is_corresponding":false,"raw_author_name":"Kosmas Pinitas","raw_affiliation_strings":["Institute of Digital Games, University of Malta,Msida,Malta","Institute of Digital Games, University of Malta, Msida, Malta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Digital Games, University of Malta,Msida,Malta","institution_ids":["https://openalex.org/I197854408"]},{"raw_affiliation_string":"Institute of Digital Games, University of Malta, Msida, Malta","institution_ids":["https://openalex.org/I197854408"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081272468","display_name":"Emmanouil Xylakis","orcid":"https://orcid.org/0000-0001-5235-0772"},"institutions":[{"id":"https://openalex.org/I197854408","display_name":"University of Malta","ror":"https://ror.org/03a62bv60","country_code":"MT","type":"education","lineage":["https://openalex.org/I197854408"]}],"countries":["MT"],"is_corresponding":false,"raw_author_name":"Emmanouil Xylakis","raw_affiliation_strings":["Institute of Digital Games, University of Malta,Msida,Malta","Institute of Digital Games, University of Malta, Msida, Malta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Digital Games, University of Malta,Msida,Malta","institution_ids":["https://openalex.org/I197854408"]},{"raw_affiliation_string":"Institute of Digital Games, University of Malta, Msida, Malta","institution_ids":["https://openalex.org/I197854408"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057445534","display_name":"Konstantinos Makantasis","orcid":"https://orcid.org/0000-0002-0889-2766"},"institutions":[{"id":"https://openalex.org/I197854408","display_name":"University of Malta","ror":"https://ror.org/03a62bv60","country_code":"MT","type":"education","lineage":["https://openalex.org/I197854408"]}],"countries":["MT"],"is_corresponding":false,"raw_author_name":"Konstantinos Makantasis","raw_affiliation_strings":["Institute of Digital Games, University of Malta,Msida,Malta","Institute of Digital Games, University of Malta, Msida, Malta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Digital Games, University of Malta,Msida,Malta","institution_ids":["https://openalex.org/I197854408"]},{"raw_affiliation_string":"Institute of Digital Games, University of Malta, Msida, Malta","institution_ids":["https://openalex.org/I197854408"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012075090","display_name":"Antonios Liapis","orcid":"https://orcid.org/0000-0001-5554-1961"},"institutions":[{"id":"https://openalex.org/I197854408","display_name":"University of Malta","ror":"https://ror.org/03a62bv60","country_code":"MT","type":"education","lineage":["https://openalex.org/I197854408"]}],"countries":["MT"],"is_corresponding":false,"raw_author_name":"Antonios Liapis","raw_affiliation_strings":["Institute of Digital Games, University of Malta,Msida,Malta","Institute of Digital Games, University of Malta, Msida, Malta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Digital Games, University of Malta,Msida,Malta","institution_ids":["https://openalex.org/I197854408"]},{"raw_affiliation_string":"Institute of Digital Games, University of Malta, Msida, Malta","institution_ids":["https://openalex.org/I197854408"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102978041","display_name":"Georgios N. Yannakakis","orcid":"https://orcid.org/0000-0003-2728-4026"},"institutions":[{"id":"https://openalex.org/I197854408","display_name":"University of Malta","ror":"https://ror.org/03a62bv60","country_code":"MT","type":"education","lineage":["https://openalex.org/I197854408"]}],"countries":["MT"],"is_corresponding":false,"raw_author_name":"Georgios N. Yannakakis","raw_affiliation_strings":["Institute of Digital Games, University of Malta,Msida,Malta","Institute of Digital Games, University of Malta, Msida, Malta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Digital Games, University of Malta,Msida,Malta","institution_ids":["https://openalex.org/I197854408"]},{"raw_affiliation_string":"Institute of Digital Games, University of Malta, Msida, Malta","institution_ids":["https://openalex.org/I197854408"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I197854408"],"apc_list":null,"apc_paid":null,"fwci":0.9678,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.81298959,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9725000262260437,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/affect","display_name":"Affect (linguistics)","score":0.7585095167160034},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6912014484405518},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6371445655822754},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.626875102519989},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.49919795989990234},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.37349840998649597},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3215596675872803},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.24397844076156616},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10082170367240906}],"concepts":[{"id":"https://openalex.org/C2776035688","wikidata":"https://www.wikidata.org/wiki/Q1606558","display_name":"Affect (linguistics)","level":2,"score":0.7585095167160034},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6912014484405518},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6371445655822754},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.626875102519989},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.49919795989990234},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.37349840998649597},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3215596675872803},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.24397844076156616},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10082170367240906},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/aciiw59127.2023.10388103","is_oa":false,"landing_page_url":"https://doi.org/10.1109/aciiw59127.2023.10388103","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 11th International Conference on Affective Computing and Intelligent Interaction Workshops and Demos (ACIIW)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Decent work and economic growth","score":0.6499999761581421,"id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320321039","display_name":"Malta Council for Science and Technology","ror":"https://ror.org/05qk3dq04"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W2047573229","https://openalex.org/W2053154970","https://openalex.org/W2087748929","https://openalex.org/W2132555391","https://openalex.org/W2145598468","https://openalex.org/W2159306398","https://openalex.org/W2181873835","https://openalex.org/W2532183443","https://openalex.org/W2587982884","https://openalex.org/W2592535880","https://openalex.org/W2604237755","https://openalex.org/W2787552263","https://openalex.org/W2787783313","https://openalex.org/W2787876196","https://openalex.org/W2900358852","https://openalex.org/W2910165986","https://openalex.org/W2994890005","https://openalex.org/W3094518473","https://openalex.org/W4210764005","https://openalex.org/W4293342342","https://openalex.org/W4362681517","https://openalex.org/W6628256265","https://openalex.org/W6630447061","https://openalex.org/W6735859517"],"related_works":["https://openalex.org/W2033512842","https://openalex.org/W4322734194","https://openalex.org/W4233600955","https://openalex.org/W2913665393","https://openalex.org/W2369695847","https://openalex.org/W3005535424","https://openalex.org/W2994319598","https://openalex.org/W2047067935","https://openalex.org/W1607054433","https://openalex.org/W2110842462"],"abstract_inverted_index":{"The":[0,165],"laborious":[1],"and":[2,20,66,73,153,155,171],"costly":[3],"nature":[4],"of":[5,28,99,103,125,141,159],"affect":[6,22,143],"annotation":[7,48,53,176],"is":[8,169],"a":[9,71,96],"key":[10],"detrimental":[11],"factor":[12],"for":[13,45],"obtaining":[14],"large":[15],"scale":[16],"corpora":[17],"with":[18,58,145],"valid":[19],"reliable":[21,121],"labels.":[23],"Motivated":[24],"by":[25,95],"the":[26,52,80,92,101,110,123,132,139,157,174],"lack":[27],"tools":[29],"that":[30,51,86,109,116],"can":[31,136],"effectively":[32,138],"determine":[33],"an":[34,74,142],"annotator\u2019s":[35],"reliability,":[36],"this":[37],"paper":[38],"proposes":[39],"general":[40],"quality":[41],"assurance":[42],"(QA)":[43],"tests":[44],"real-time":[46],"continuous":[47],"tasks.":[49],"Assuming":[50],"tasks":[54],"rely":[55],"on":[56,150],"stimuli":[57],"audiovisual":[59],"components,":[60],"such":[61],"as":[62],"videos,":[63],"we":[64,128],"propose":[65],"evaluate":[67],"two":[68],"QA":[69,76,81,112,133,167],"tests:":[70],"visual":[72],"auditory":[75],"test.":[77],"We":[78],"validate":[79],"tool":[82,113,134,168],"across":[83],"20":[84],"annotators":[85,118],"are":[87,119],"asked":[88],"to":[89],"go":[90],"through":[91,173],"test":[93],"followed":[94],"lengthy":[97],"task":[98],"annotating":[100],"engagement":[102],"gameplay":[104],"videos.":[105],"Our":[106],"findings":[107],"suggest":[108],"proposed":[111],"reveals,":[114],"unsurprisingly,":[115],"trained":[117],"more":[120],"than":[122],"best":[124],"untrained":[126],"crowdworkers":[127],"could":[129],"employ.":[130],"Importantly,":[131],"introduced":[135,166],"predict":[137],"reliability":[140,158],"annotator":[144],"80%":[146],"accuracy,":[147],"thereby,":[148],"saving":[149],"resources,":[151],"effort":[152],"cost,":[154],"maximizing":[156],"labels":[160],"solicited":[161],"in":[162],"affective":[163],"corpora.":[164],"available":[170],"accessible":[172],"PAGAN":[175],"platform.":[177]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":3}],"updated_date":"2026-06-26T08:34:08.712188","created_date":"2025-10-10T00:00:00"}