{"id":"https://openalex.org/W7026719218","doi":null,"title":"Analyzing the Wild-West of Interrater Agreement in Affective Content Analysis on Text: A Systematic Literature Review","display_name":"Analyzing the Wild-West of Interrater Agreement in Affective Content Analysis on Text: A Systematic Literature Review","alternate_titles":[],"publication_year":2024,"publication_date":"2024-06-27","ids":{"openalex":"https://openalex.org/W7026719218"},"language":"en","primary_location":{"id":"pmh:oai:tudelft.nl:uuid:99ac2d78-71de-4b2c-be4e-10d8129eac5a","doi":null,"is_oa":true,"landing_page_url":"http://resolver.tudelft.nl/uuid:99ac2d78-71de-4b2c-be4e-10d8129eac5a","pdf_url":"https://repository.tudelft.nl/file/File_3c0347a6-84d0-49ce-95b4-eea0bae0937a","source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"bachelor thesis"},"parent_works":[],"lineage":["https://openalex.org/W7026719218"],"child_works_count":0,"type":"review","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://repository.tudelft.nl/file/File_3c0347a6-84d0-49ce-95b4-eea0bae0937a","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Macsim, Violeta (author)","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"contributor_roles":[],"raw_author_name":"Macsim, Violeta (author)","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0196,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.0836039,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":87,"max":92},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.6255000233650208,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.6255000233650208,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.3700999915599823,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.0010000000474974513,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/interobserver-agreement","display_name":"interobserver agreement","score":0.9639999866485596},{"id":"https://openalex.org/keywords/annotation-consistency","display_name":"annotation consistency","score":0.5640000104904175},{"id":"https://openalex.org/keywords/annotated-data","display_name":"annotated dataset","score":0.6209999918937683},{"id":"https://openalex.org/keywords/manual-annotation","display_name":"manual annotation","score":0.4059999883174896},{"id":"https://openalex.org/keywords/annotation-standardization","display_name":"annotation standardization","score":0.6809999942779541}],"concepts":[{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.7567999958992004},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6111999750137329},{"id":"https://openalex.org/C162446236","wikidata":"https://www.wikidata.org/wiki/Q653137","display_name":"Content analysis","level":2,"score":0.5382000207901001},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5149999856948853},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.49869999289512634},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4959000051021576},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4763999879360199},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.474700003862381},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.4458000063896179},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.4138000011444092},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.4115999937057495},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.39820000529289246},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38119998574256897},{"id":"https://openalex.org/C95190672","wikidata":"https://www.wikidata.org/wiki/Q815382","display_name":"Meta-analysis","level":2,"score":0.37770000100135803},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3720000088214874},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.35839998722076416},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3142000138759613},{"id":"https://openalex.org/C2778152352","wikidata":"https://www.wikidata.org/wiki/Q5165061","display_name":"Content (measure theory)","level":2,"score":0.3124000132083893},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.28290000557899475},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.2806999981403351},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.2806999981403351},{"id":"https://openalex.org/C2776035688","wikidata":"https://www.wikidata.org/wiki/Q1606558","display_name":"Affect (linguistics)","level":2,"score":0.26570001244544983},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.25589999556541443}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:tudelft.nl:uuid:99ac2d78-71de-4b2c-be4e-10d8129eac5a","doi":null,"is_oa":true,"landing_page_url":"http://resolver.tudelft.nl/uuid:99ac2d78-71de-4b2c-be4e-10d8129eac5a","pdf_url":"https://repository.tudelft.nl/file/File_3c0347a6-84d0-49ce-95b4-eea0bae0937a","source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"bachelor thesis"}],"best_oa_location":{"id":"pmh:oai:tudelft.nl:uuid:99ac2d78-71de-4b2c-be4e-10d8129eac5a","doi":null,"is_oa":true,"landing_page_url":"http://resolver.tudelft.nl/uuid:99ac2d78-71de-4b2c-be4e-10d8129eac5a","pdf_url":"https://repository.tudelft.nl/file/File_3c0347a6-84d0-49ce-95b4-eea0bae0937a","source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"bachelor thesis"},"sustainable_development_goals":[],"study_designs":[{"display_name":"Systematic Review","id":"https://openalex.org/study-designs/systematic-review"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7026719218.pdf","grobid_xml":"https://content.openalex.org/works/W7026719218.grobid-xml"},"referenced_works_count":82,"referenced_works":["https://openalex.org/W1513398909","https://openalex.org/W1523983350","https://openalex.org/W1525983470","https://openalex.org/W1595791987","https://openalex.org/W1966797434","https://openalex.org/W1972095489","https://openalex.org/W1975879668","https://openalex.org/W2035866663","https://openalex.org/W2049988375","https://openalex.org/W2053154970","https://openalex.org/W2102998034","https://openalex.org/W2104190448","https://openalex.org/W2105468141","https://openalex.org/W2112422413","https://openalex.org/W2128837546","https://openalex.org/W2131915193","https://openalex.org/W2133943399","https://openalex.org/W2146907761","https://openalex.org/W2150617068","https://openalex.org/W2182096631","https://openalex.org/W2184066413","https://openalex.org/W2287528930","https://openalex.org/W2294703018","https://openalex.org/W2402365003","https://openalex.org/W2580843878","https://openalex.org/W2618843390","https://openalex.org/W2741033833","https://openalex.org/W2747541555","https://openalex.org/W2752234108","https://openalex.org/W2758435862","https://openalex.org/W2798690845","https://openalex.org/W2805744755","https://openalex.org/W2810414047","https://openalex.org/W2874464011","https://openalex.org/W2884442519","https://openalex.org/W2886486953","https://openalex.org/W2907095334","https://openalex.org/W2917725866","https://openalex.org/W2922411907","https://openalex.org/W2955429306","https://openalex.org/W2990731097","https://openalex.org/W3012159372","https://openalex.org/W3012385961","https://openalex.org/W3015947791","https://openalex.org/W3021162616","https://openalex.org/W3022113899","https://openalex.org/W3027721233","https://openalex.org/W3035720554","https://openalex.org/W3081464307","https://openalex.org/W3115932868","https://openalex.org/W3118615836","https://openalex.org/W3159239236","https://openalex.org/W3162471890","https://openalex.org/W3164079775","https://openalex.org/W3180117439","https://openalex.org/W3194405061","https://openalex.org/W3194919258","https://openalex.org/W3196324893","https://openalex.org/W3214186028","https://openalex.org/W4205973286","https://openalex.org/W4210811935","https://openalex.org/W4220887861","https://openalex.org/W4231519654","https://openalex.org/W4256441345","https://openalex.org/W4283641455","https://openalex.org/W4285395379","https://openalex.org/W4308764932","https://openalex.org/W4311970277","https://openalex.org/W4312159272","https://openalex.org/W4312914062","https://openalex.org/W4318067823","https://openalex.org/W4382052522","https://openalex.org/W4382406715","https://openalex.org/W4385194642","https://openalex.org/W4386798605","https://openalex.org/W4389158411","https://openalex.org/W4389788950","https://openalex.org/W4390872056","https://openalex.org/W4390874746","https://openalex.org/W4391021087","https://openalex.org/W7070580227","https://openalex.org/W7131799771"],"related_works":["https://openalex.org/W7034079611","https://openalex.org/W7071674733","https://openalex.org/W7039857910","https://openalex.org/W2558525879","https://openalex.org/W2049988375","https://openalex.org/W2000271339","https://openalex.org/W7026728410","https://openalex.org/W2011822800","https://openalex.org/W302528027","https://openalex.org/W3156613214"],"abstract_inverted_index":{"Human-computer":[0],"interaction":[1],"has":[2,85],"long":[3],"been":[4],"the":[5,50,79,97,100,108,149,158,213,233,241,258,262,269,279],"focus":[6,191],"of":[7,16,71,81,99,121,167,187,201,208,235,264,271,281],"technological":[8],"evolution;":[9],"however,":[10],"in":[11,78,89,95,107,132,205],"order":[12],"for":[13,151],"this":[14,221,276,289],"type":[15],"system":[17],"to":[18,41,48,60,139,147,212,219,227,239,252,266],"reach":[19],"its":[20],"peak":[21,88],"potential,":[22],"machines":[23],"must":[24],"recognize":[25],"that":[26,197],"humans":[27,104,216],"are":[28,38,57,198,217,224,250],"constantly":[29],"influenced":[30],"by":[31,53,103],"emotions.":[32],"Text":[33],"affective":[34,74],"content":[35],"analysis":[36,207],"models":[37,168],"one":[39],"attempt":[40],"integrate":[42],"human":[43],"psychology":[44],"into":[45],"computers,":[46],"trying":[47],"detect":[49],"emotion":[51],"transmitted":[52],"written":[54,195],"input.":[55],"There":[56],"numerous":[58],"approaches":[59],"implementing":[61],"such":[62],"systems,":[63,203],"with":[64,189],"supervised":[65],"learning":[66],"still":[67],"popular.":[68],"The":[69,124,143,173,247],"challenge":[70],"creating":[72],"textual":[73],"datasets":[75],"is":[76,146],"not":[77,199],"availability":[80],"records,":[82],"as":[83,127,129],"humanity":[84],"reached":[86],"a":[87,113,190,274],"data":[90],"production,":[91],"especially":[92],"text,":[93],"but":[94],"ensuring":[96],"consistency":[98],"annotations":[101],"provided":[102],"when":[105,215],"included":[106],"process.":[109],"This":[110],"study":[111,154],"conducts":[112],"systematic":[114],"literature":[115,175],"review":[116],"focused":[117],"on":[118,170,192],"providing":[119],"details":[120],"published":[122],"corpora.":[123],"annotation":[125,282],"process,":[126],"well":[128],"any":[130],"trends":[131],"how":[133],"it":[134],"evolved,":[135],"will":[136],"be":[137],"examined":[138],"obtain":[140],"dataset":[141],"particularities.":[142],"ultimate":[144],"intent":[145],"lay":[148],"groundwork":[150],"an":[152,206],"ample":[153],"aimed":[155],"at":[156],"analyzing":[157],"relationship":[159],"between":[160,237],"interrater":[161],"agreement":[162,236],"levels":[163],"and":[164,185,231,285],"performance":[165],"scores":[166],"trained":[169],"these":[171,255],"datasets.":[172,210],"relevant":[174],"was":[176],"extracted":[177],"from":[178],"3":[179],"search":[180],"engines:":[181],"Scopus,":[182],"IEEE":[183],"Xplore,":[184],"Web":[186],"Science,":[188],"manually":[193],"labeled":[194],"records":[196],"part":[200],"multimodal":[202],"resulting":[204],"41":[209],"According":[211],"aggregations,":[214],"recruited":[218],"perform":[220],"task,":[222],"researchers":[223,249],"more":[225],"likely":[226],"use":[228],"multiple":[229],"annotators":[230],"calculate":[232],"degree":[234],"them":[238],"ensure":[240],"data's":[242],"reliability":[243],"before":[244,257],"using":[245],"it.":[246],"conducting":[248],"inclined":[251],"either":[253],"train":[254],"people":[256],"procedure":[259],"or":[260],"tailor":[261],"set":[263],"labels":[265],"potentially":[267],"increase":[268],"uniformity":[270],"ratings.":[272],"As":[273],"result,":[275],"paper":[277],"highlights":[278],"variety":[280],"process":[283],"characteristics":[284],"points":[286],"towards":[287],"standardizing":[288],"task.":[290]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7026719218","counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2025-10-10T00:00:00"}
