{"id":"https://openalex.org/W2902985622","doi":"https://doi.org/10.1109/snams.2018.8554744","title":"Inter-Rater Agreement for Social Computing Studies","display_name":"Inter-Rater Agreement for Social Computing Studies","publication_year":2018,"publication_date":"2018-10-01","ids":{"openalex":"https://openalex.org/W2902985622","doi":"https://doi.org/10.1109/snams.2018.8554744","mag":"2902985622"},"language":"en","primary_location":{"id":"doi:10.1109/snams.2018.8554744","is_oa":false,"landing_page_url":"https://doi.org/10.1109/snams.2018.8554744","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2018 Fifth International Conference on Social Networks Analysis, Management and Security (SNAMS)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083126896","display_name":"Joni Salminen","orcid":"https://orcid.org/0000-0003-3230-0561"},"institutions":[{"id":"https://openalex.org/I4210144839","display_name":"Hamad bin Khalifa University","ror":"https://ror.org/03eyq4y97","country_code":"QA","type":"education","lineage":["https://openalex.org/I4210144839"]}],"countries":["QA"],"is_corresponding":true,"raw_author_name":"Joni O. Salminen","raw_affiliation_strings":["Qatar Computing Research Institute, Hamad Bin Khalifa University, Doha, Qatar"],"affiliations":[{"raw_affiliation_string":"Qatar Computing Research Institute, Hamad Bin Khalifa University, Doha, Qatar","institution_ids":["https://openalex.org/I4210144839"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077958415","display_name":"Hind Almerekhi","orcid":"https://orcid.org/0000-0002-3183-2374"},"institutions":[{"id":"https://openalex.org/I4210144839","display_name":"Hamad bin Khalifa University","ror":"https://ror.org/03eyq4y97","country_code":"QA","type":"education","lineage":["https://openalex.org/I4210144839"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Hind A. Al-Merekhi","raw_affiliation_strings":["Qatar Computing Research Institute, Hamad Bin Khalifa University, Doha, Qatar"],"affiliations":[{"raw_affiliation_string":"Qatar Computing Research Institute, Hamad Bin Khalifa University, Doha, Qatar","institution_ids":["https://openalex.org/I4210144839"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102708991","display_name":"Partha Pratim Dey","orcid":"https://orcid.org/0000-0002-8093-8802"},"institutions":[{"id":"https://openalex.org/I4210145126","display_name":"Aditya Birla (India)","ror":"https://ror.org/03pztks36","country_code":"IN","type":"company","lineage":["https://openalex.org/I4210145126"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Partha Dey","raw_affiliation_strings":["Department of Mechanical Engineering, Academy of Technology, Adisaptagram, India"],"affiliations":[{"raw_affiliation_string":"Department of Mechanical Engineering, Academy of Technology, Adisaptagram, India","institution_ids":["https://openalex.org/I4210145126"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5044849571","display_name":"Bernard J. Jansen","orcid":"https://orcid.org/0000-0002-6468-6609"},"institutions":[{"id":"https://openalex.org/I4210144839","display_name":"Hamad bin Khalifa University","ror":"https://ror.org/03eyq4y97","country_code":"QA","type":"education","lineage":["https://openalex.org/I4210144839"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Bernard J. Jansen","raw_affiliation_strings":["Qatar Computing Research Institute, Hamad Bin Khalifa University, Doha, Qatar"],"affiliations":[{"raw_affiliation_string":"Qatar Computing Research Institute, Hamad Bin Khalifa University, Doha, Qatar","institution_ids":["https://openalex.org/I4210144839"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5083126896"],"corresponding_institution_ids":["https://openalex.org/I4210144839"],"apc_list":null,"apc_paid":null,"fwci":3.4665,"has_fulltext":false,"cited_by_count":23,"citation_normalized_percentile":{"value":0.92965747,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"80","last_page":"87"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10064","display_name":"Complex Network Analysis Techniques","score":0.9789999723434448,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.948199987411499,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8999646902084351},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.7226289510726929},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6860141754150391},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"Inter-rater reliability","score":0.6773641705513},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.6306644678115845},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5889742970466614},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5802909731864929},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.5702334046363831},{"id":"https://openalex.org/keywords/computational-sociology","display_name":"Computational sociology","score":0.5293679237365723},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.45816662907600403},{"id":"https://openalex.org/keywords/subjectivity","display_name":"Subjectivity","score":0.44095468521118164},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4051584303379059},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3950275182723999},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.3322996497154236},{"id":"https://openalex.org/keywords/rating-scale","display_name":"Rating scale","score":0.2554691433906555},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.0992668867111206}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8999646902084351},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7226289510726929},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6860141754150391},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.6773641705513},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.6306644678115845},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5889742970466614},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5802909731864929},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5702334046363831},{"id":"https://openalex.org/C126349790","wikidata":"https://www.wikidata.org/wiki/Q905036","display_name":"Computational sociology","level":2,"score":0.5293679237365723},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.45816662907600403},{"id":"https://openalex.org/C202889954","wikidata":"https://www.wikidata.org/wiki/Q1139554","display_name":"Subjectivity","level":2,"score":0.44095468521118164},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4051584303379059},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3950275182723999},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3322996497154236},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.2554691433906555},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0992668867111206},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/snams.2018.8554744","is_oa":false,"landing_page_url":"https://doi.org/10.1109/snams.2018.8554744","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2018 Fifth International Conference on Social Networks Analysis, Management and Security (SNAMS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth","score":0.550000011920929}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W84786028","https://openalex.org/W1975879668","https://openalex.org/W1983897914","https://openalex.org/W1996290308","https://openalex.org/W2019029324","https://openalex.org/W2021097538","https://openalex.org/W2025862520","https://openalex.org/W2031738304","https://openalex.org/W2048601276","https://openalex.org/W2053154970","https://openalex.org/W2091950237","https://openalex.org/W2133943399","https://openalex.org/W2136400720","https://openalex.org/W2141124745","https://openalex.org/W2141766660","https://openalex.org/W2153072998","https://openalex.org/W2157035150","https://openalex.org/W2340954483","https://openalex.org/W2346914605","https://openalex.org/W2394682549","https://openalex.org/W2396488496","https://openalex.org/W2524874498","https://openalex.org/W2548105042","https://openalex.org/W2763236539","https://openalex.org/W2789642357","https://openalex.org/W2884803076","https://openalex.org/W6712042790"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W1503094549","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W3178669050","https://openalex.org/W4390239008"],"abstract_inverted_index":{"Different":[0],"agreement":[1,26,62,98,120],"scores":[2,166],"are":[3,39,54,64,167],"widely":[4],"used":[5],"in":[6,33],"social":[7,35,50,100,197],"computing":[8,51,101],"studies":[9],"to":[10,109,170,177,186],"evaluate":[11],"the":[12,23,61,76,79,82,84,97,106,117,128,136,148,164,171,178,183,187],"reliability":[13,165],"of":[14,25,99,119,127,130,138,150,157,180,189],"crowdsourced":[15],"ratings.":[16],"In":[17],"this":[18,45],"research,":[19],"we":[20],"argue":[21],"that":[22,53,60,75,87,95,163],"concept":[24],"is":[27],"problematic":[28],"for":[29,195,201],"many":[30],"rating":[31,80],"tasks":[32,88,102],"computational":[34,196],"science":[36],"because":[37],"they":[38,134],"characterized":[40],"by":[41,47,56,90],"subjectivity.":[42],"We":[43,142],"demonstrate":[44],"claim":[46],"analyzing":[48],"four":[49],"datasets":[52],"rated":[55],"crowd":[57,131,151],"workers,":[58,132],"showing":[59],"ratings":[63,152],"low":[65],"despite":[66],"deploying":[67],"proper":[68],"instructions":[69],"and":[70,94,160,182],"platform":[71],"settings.":[72],"Findings":[73],"indicate":[74],"more":[77],"subjective":[78,115],"task,":[81],"lower":[83],"agreement,":[85],"suggesting":[86],"differ":[89],"their":[91],"inherent":[92],"subjectivity":[93],"measuring":[96],"might":[103],"not":[104],"be":[105],"optimal":[107],"way":[108],"ensure":[110],"data":[111,202],"quality.":[112],"When":[113],"creating":[114],"tasks,":[116],"use":[118],"metrics":[121],"potentially":[122],"gives":[123],"a":[124,154],"false":[125],"picture":[126],"consistency":[129],"as":[133],"over-simplify":[135],"reality":[137],"obtaining":[139],"quality":[140],"labels.":[141],"also":[143],"provide":[144],"empirical":[145],"evidence":[146],"on":[147],"stability":[149],"with":[153],"different":[155],"number":[156,172,179,188],"raters,":[158,181],"items,":[159],"categories,":[161,173],"finding":[162],"most":[168],"sensitive":[169,176,185],"somewhat":[174],"less":[175],"least":[184],"items.":[190],"Our":[191],"findings":[192],"have":[193],"implications":[194],"scientists":[198],"using":[199],"crowdsourcing":[200],"collection.":[203]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":7},{"year":2019,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}