{"id":"https://openalex.org/W6929281068","doi":"https://doi.org/10.48340/ecscw2021_n12","title":"The Problem of Majority Voting in Crowdsourcing with Binary Classes","display_name":"The Problem of Majority Voting in Crowdsourcing with Binary Classes","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W6929281068","doi":"https://doi.org/10.48340/ecscw2021_n12"},"language":"en","primary_location":{"id":"doi:10.48340/ecscw2021_n12","is_oa":true,"landing_page_url":"https://doi.org/10.48340/ecscw2021_n12","pdf_url":null,"source":{"id":"https://openalex.org/S7407053554","display_name":"European Society for Socially Embedded Technologies (EUSSETDL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48340/ecscw2021_n12","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Salminen, Joni","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Salminen, Joni","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Kamel, Ahmed Mohamed","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kamel, Ahmed Mohamed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Jung, Soon-Gyo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jung, Soon-Gyo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Jansen, Bernard","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jansen, Bernard","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.40375638,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.0010000000474974513,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.0005000000237487257,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9082000255584717},{"id":"https://openalex.org/keywords/majority-rule","display_name":"Majority rule","score":0.8087000250816345},{"id":"https://openalex.org/keywords/voting","display_name":"Voting","score":0.5989999771118164},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.5331000089645386},{"id":"https://openalex.org/keywords/property","display_name":"Property (philosophy)","score":0.5273000001907349},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.5188999772071838}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9082000255584717},{"id":"https://openalex.org/C153668964","wikidata":"https://www.wikidata.org/wiki/Q27636","display_name":"Majority rule","level":2,"score":0.8087000250816345},{"id":"https://openalex.org/C520049643","wikidata":"https://www.wikidata.org/wiki/Q189760","display_name":"Voting","level":3,"score":0.5989999771118164},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.5331000089645386},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.5273000001907349},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.5188999772071838},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.47119998931884766},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.47029998898506165},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46209999918937683},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4422000050544739},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.43880000710487366},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.39750000834465027},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.35260000824928284},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.33469998836517334},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.31839999556541443},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.3124000132083893},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.3070000112056732},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2937999963760376},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2554999887943268},{"id":"https://openalex.org/C3020318244","wikidata":"https://www.wikidata.org/wiki/Q4812187","display_name":"Large sample","level":2,"score":0.25209999084472656}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48340/ecscw2021_n12","is_oa":true,"landing_page_url":"https://doi.org/10.48340/ecscw2021_n12","pdf_url":null,"source":{"id":"https://openalex.org/S7407053554","display_name":"European Society for Socially Embedded Technologies (EUSSETDL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"doi:10.48340/ecscw2021_n12","is_oa":true,"landing_page_url":"https://doi.org/10.48340/ecscw2021_n12","pdf_url":null,"source":{"id":"https://openalex.org/S7407053554","display_name":"European Society for Socially Embedded Technologies (EUSSETDL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.728285551071167}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"When":[0],"there":[1,142],"are":[2],"two":[3],"classes,":[4],"a":[5,22,35,90,125,144,154],"majority":[6],"vote":[7],"can":[8,16],"always":[9],"be":[10,122],"obtained":[11],"with":[12,37,47,139],"three":[13,80],"labelers.":[14],"Researchers":[15],"utilize":[17],"this":[18],"property":[19],"to":[20,83,128,166],"obtain":[21],"false":[23],"sense":[24],"of":[25,97,103,107,119,131,148,157,170],"confidence":[26],"in":[27,66,133],"their":[28],"ground":[29],"truth":[30],"labels.":[31],"We":[32,69,151],"demonstrate":[33],"such":[34],"case":[36],"3000":[38],"crowdsourced":[39],"labels":[40],"for":[41,109],"an":[42],"online":[43],"hate":[44],"dataset.":[45],"Evaluating":[46],"percentage":[48],"agreement,":[49],"Gwet\u2019s":[50],"AC1,":[51],"and":[52],"Krippendorff\u2019s":[53],"alpha,":[54],"results":[55],"show":[56,70],"that":[57,71],"using":[58],"more":[59,164],"raters":[60,76,81,85,88,108,120,165],"teases":[61],"out":[62],"the":[63,75,101,105,117,129,158,168,173],"hidden":[64],"nuances":[65],"raters\u2019":[67],"preferences.":[68],"full":[72],"agreement":[73,92,132,171],"among":[74,172],"monotonically":[77],"decreases":[78],"from":[79],"(28.4%)":[82],"nine":[84],"(19.5%).":[86],"Ten":[87],"have":[89],"higher":[91],"than":[93],"any":[94],"other":[95],"number":[96,106,118,147],"raters,":[98,141],"which":[99],"supports":[100],"idea":[102],"increasing":[104,116],"subjective":[110,134],"labeling":[111],"tasks.":[112],"Nevertheless,":[113],"while":[114],"beneficial,":[115],"cannot":[121],"considered":[123],"as":[124,137],"fundamental":[126],"solution":[127],"issue":[130],"crowdsourcing":[135],"tasks,":[136],"even":[138],"ten":[140],"is":[143],"non-":[145],"negligible":[146],"ties":[149],"(4.11%).":[150],"suggest":[152],"having":[153],"small":[155],"sample":[156],"data":[159],"labeled":[160],"by":[161],"five":[162],"or":[163],"evaluate":[167],"stability":[169],"raters.":[174]},"counts_by_year":[],"updated_date":"2025-11-06T06:51:31.235846","created_date":"2025-10-10T00:00:00"}