{"id":"https://openalex.org/W4402441570","doi":"https://doi.org/10.1145/3694980","title":"A Survey on the Role of Crowds in Combating Online Misinformation: Annotators, Evaluators, and Creators","display_name":"A Survey on the Role of Crowds in Combating Online Misinformation: Annotators, Evaluators, and Creators","publication_year":2024,"publication_date":"2024-09-11","ids":{"openalex":"https://openalex.org/W4402441570","doi":"https://doi.org/10.1145/3694980"},"language":"en","primary_location":{"id":"doi:10.1145/3694980","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3694980","pdf_url":null,"source":{"id":"https://openalex.org/S41523882","display_name":"ACM Transactions on Knowledge Discovery from Data","issn_l":"1556-4681","issn":["1556-4681","1556-472X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Knowledge Discovery from Data","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039926801","display_name":"Bing He","orcid":"https://orcid.org/0000-0002-4369-1150"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bing He","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA","Georgia Institute of Technology, USA"],"raw_orcid":"https://orcid.org/0000-0002-4369-1150","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]},{"raw_affiliation_string":"Georgia Institute of Technology, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5097055353","display_name":"Yibo Hu","orcid":"https://orcid.org/0000-0002-1578-7892"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yibo Hu","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA","Georgia Institute of Technology, USA"],"raw_orcid":"https://orcid.org/0000-0002-1578-7892","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]},{"raw_affiliation_string":"Georgia Institute of Technology, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028740930","display_name":"Yeon-Chang Lee","orcid":"https://orcid.org/0000-0002-8769-0678"},"institutions":[{"id":"https://openalex.org/I48566637","display_name":"Ulsan National Institute of Science and Technology","ror":"https://ror.org/017cjz748","country_code":"KR","type":"education","lineage":["https://openalex.org/I48566637"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Yeon-Chang Lee","raw_affiliation_strings":["Ulsan National Institute of Science and Technology (UNIST), Ulsan, Korea","Ulsan National Institute of Science and Technology (UNIST), Korea"],"raw_orcid":"https://orcid.org/0000-0002-8769-0678","affiliations":[{"raw_affiliation_string":"Ulsan National Institute of Science and Technology (UNIST), Ulsan, Korea","institution_ids":["https://openalex.org/I48566637"]},{"raw_affiliation_string":"Ulsan National Institute of Science and Technology (UNIST), Korea","institution_ids":["https://openalex.org/I48566637"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022236610","display_name":"Soyoung Oh","orcid":"https://orcid.org/0000-0002-6466-686X"},"institutions":[{"id":"https://openalex.org/I91712215","display_name":"Saarland University","ror":"https://ror.org/01jdpyv68","country_code":"DE","type":"education","lineage":["https://openalex.org/I91712215"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Soyoung Oh","raw_affiliation_strings":["Saarland University, Saarbrucken, Germany","Saarland University, Germany"],"raw_orcid":"https://orcid.org/0000-0002-6466-686X","affiliations":[{"raw_affiliation_string":"Saarland University, Saarbrucken, Germany","institution_ids":["https://openalex.org/I91712215"]},{"raw_affiliation_string":"Saarland University, Germany","institution_ids":["https://openalex.org/I91712215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085250715","display_name":"Gaurav Verma","orcid":"https://orcid.org/0000-0001-6182-9857"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gaurav Verma","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA","Georgia Institute of Technology, USA"],"raw_orcid":"https://orcid.org/0000-0001-6182-9857","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]},{"raw_affiliation_string":"Georgia Institute of Technology, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056142478","display_name":"Srijan Kumar","orcid":"https://orcid.org/0000-0002-5796-3532"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Srijan Kumar","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA","Georgia Institute of Technology, USA"],"raw_orcid":"https://orcid.org/0000-0002-5796-3532","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]},{"raw_affiliation_string":"Georgia Institute of Technology, USA","institution_ids":["https://openalex.org/I130701444"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.9685,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.97994553,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"19","issue":"1","first_page":"1","last_page":"30"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/misinformation","display_name":"Misinformation","score":0.9295138120651245},{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.8395112752914429},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6267305612564087},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.43102723360061646},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4034833312034607},{"id":"https://openalex.org/keywords/internet-privacy","display_name":"Internet privacy","score":0.3612791895866394},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2737482190132141},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.12676066160202026}],"concepts":[{"id":"https://openalex.org/C2776990098","wikidata":"https://www.wikidata.org/wiki/Q13579947","display_name":"Misinformation","level":2,"score":0.9295138120651245},{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.8395112752914429},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6267305612564087},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.43102723360061646},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4034833312034607},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.3612791895866394},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2737482190132141},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.12676066160202026}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3694980","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3694980","pdf_url":null,"source":{"id":"https://openalex.org/S41523882","display_name":"ACM Transactions on Knowledge Discovery from Data","issn_l":"1556-4681","issn":["1556-4681","1556-472X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Knowledge Discovery from Data","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":112,"referenced_works":["https://openalex.org/W1054048471","https://openalex.org/W2001642679","https://openalex.org/W2005501262","https://openalex.org/W2211668096","https://openalex.org/W2320433003","https://openalex.org/W2531542534","https://openalex.org/W2553232295","https://openalex.org/W2588505903","https://openalex.org/W2625555835","https://openalex.org/W2724964366","https://openalex.org/W2736043705","https://openalex.org/W2754477858","https://openalex.org/W2755238919","https://openalex.org/W2766795866","https://openalex.org/W2790166049","https://openalex.org/W2790647187","https://openalex.org/W2800334901","https://openalex.org/W2804104502","https://openalex.org/W2901432156","https://openalex.org/W2901881888","https://openalex.org/W2911715381","https://openalex.org/W2912249915","https://openalex.org/W2913081045","https://openalex.org/W2913841425","https://openalex.org/W2914872915","https://openalex.org/W2917551198","https://openalex.org/W2925538271","https://openalex.org/W2947803393","https://openalex.org/W2955681503","https://openalex.org/W2959736309","https://openalex.org/W2969631841","https://openalex.org/W2974087526","https://openalex.org/W2980473392","https://openalex.org/W3019597125","https://openalex.org/W3031781733","https://openalex.org/W3032691658","https://openalex.org/W3035065059","https://openalex.org/W3037448378","https://openalex.org/W3039592278","https://openalex.org/W3040472729","https://openalex.org/W3041140901","https://openalex.org/W3080732897","https://openalex.org/W3089703818","https://openalex.org/W3092729016","https://openalex.org/W3093549971","https://openalex.org/W3093707318","https://openalex.org/W3093940086","https://openalex.org/W3095379838","https://openalex.org/W3095892608","https://openalex.org/W3097590906","https://openalex.org/W3098992812","https://openalex.org/W3106841467","https://openalex.org/W3112473959","https://openalex.org/W3114476300","https://openalex.org/W3117449150","https://openalex.org/W3126207517","https://openalex.org/W3131324126","https://openalex.org/W3132767615","https://openalex.org/W3134705648","https://openalex.org/W3136302317","https://openalex.org/W3136503798","https://openalex.org/W3156617956","https://openalex.org/W3172233798","https://openalex.org/W3189373506","https://openalex.org/W3190075092","https://openalex.org/W3190124883","https://openalex.org/W3192678369","https://openalex.org/W3194445254","https://openalex.org/W3195126378","https://openalex.org/W3197517448","https://openalex.org/W3200542377","https://openalex.org/W3201642427","https://openalex.org/W3201783279","https://openalex.org/W3207097449","https://openalex.org/W3215246745","https://openalex.org/W3215340850","https://openalex.org/W4200022165","https://openalex.org/W4200432199","https://openalex.org/W4205805506","https://openalex.org/W4206192262","https://openalex.org/W4206341919","https://openalex.org/W4206406919","https://openalex.org/W4212782488","https://openalex.org/W4212937970","https://openalex.org/W4214581110","https://openalex.org/W4220657815","https://openalex.org/W4220710655","https://openalex.org/W4220774228","https://openalex.org/W4223573167","https://openalex.org/W4224243598","https://openalex.org/W4224457782","https://openalex.org/W4225112958","https://openalex.org/W4225361530","https://openalex.org/W4225879562","https://openalex.org/W4226158522","https://openalex.org/W4226412112","https://openalex.org/W4250637538","https://openalex.org/W4280489099","https://openalex.org/W4280600176","https://openalex.org/W4281640315","https://openalex.org/W4281675085","https://openalex.org/W4283829988","https://openalex.org/W4289912296","https://openalex.org/W4293218597","https://openalex.org/W4294214983","https://openalex.org/W4298541566","https://openalex.org/W4367046455","https://openalex.org/W4367047188","https://openalex.org/W4385257688","https://openalex.org/W4389520040","https://openalex.org/W4394880004","https://openalex.org/W7062894306"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W3197131596","https://openalex.org/W1503094549","https://openalex.org/W4390616380","https://openalex.org/W4388666321","https://openalex.org/W4205914924","https://openalex.org/W2337920774","https://openalex.org/W4283459170"],"abstract_inverted_index":{"Online":[0],"misinformation":[1,202],"poses":[2],"a":[3,117,159,220],"global":[4],"risk":[5],"with":[6],"significant":[7],"real-world":[8],"consequences.":[9],"To":[10],"combat":[11,101],"misinformation,":[12,141],"current":[13],"research":[14,43,93,279],"relies":[15],"on":[16,47],"professionals":[17],"like":[18],"journalists":[19],"and":[20,24,65,89,132,143,149,210,239,253,258,263],"fact-checkers":[21],"for":[22,35,129,207,242,256,266,277],"annotating":[23],"debunking":[25],"false":[26],"information":[27],"while":[28],"also":[29],"developing":[30],"automated":[31],"machine":[32],"learning":[33],"methods":[34],"detecting":[36],"misinformation.":[37,67,102],"Complementary":[38],"to":[39,100,112,122,140,223],"these":[40],"approaches,":[41],"recent":[42,69],"has":[44],"increasingly":[45],"concentrated":[46],"utilizing":[48],"the":[49,86,98,123,155,163,167,198,205,225,229,246],"power":[50],"of":[51,74,92,154,162,228,248],"ordinary":[52],"social":[53],"media":[54],"users,":[55],"a.k.a.":[56],"\u201cthe":[57],"crowd,\u201d":[58],"who":[59,176,184,192],"act":[60],"as":[61,174,182,190],"eyes-on-the-ground":[62],"proactively":[63],"questioning":[64],"countering":[66],"Notably,":[68],"studies":[70],"show":[71],"that":[72,95],"96%":[73],"counter-misinformation":[75,186,249],"responses":[76],"originate":[77],"from":[78],"them.":[79],"Acknowledging":[80],"their":[81],"prominent":[82],"role,":[83,217],"we":[84,106,157,218,232,269],"present":[85,136],"first":[87,107],"systematic":[88,130],"comprehensive":[90],"survey":[91,272],"papers":[94,110],"actively":[96,177],"leverage":[97],"crowds":[99,168,173,181,189],"In":[103,214],"this":[104,271,281],"survey,":[105],"identify":[108,178],"88":[109],"related":[111,139],"crowd-based":[113],"efforts,":[114],"1":[115],"following":[116],"meticulous":[118],"annotation":[119],"process":[120],"adhering":[121],"PRISMA":[124],"framework":[125],"(preferred":[126],"reporting":[127],"items":[128],"reviews":[131],"meta-analyses).":[133],"We":[134],"then":[135],"key":[137],"statistics":[138],"counter-misinformation,":[142,209],"crowd":[144],"input":[145],"in":[146,169,201,280],"different":[147],"formats":[148],"topics.":[150],"Upon":[151],"holistic":[152],"analysis":[153,222],"papers,":[156],"introduce":[158],"novel":[160],"taxonomy":[161,196],"roles":[164],"played":[165],"by":[166,273],"combating":[170],"misinformation:":[171],"(i)":[172,235],"annotators":[175],"misinformation;":[179],"(ii)":[180,244],"evaluators":[183],"assess":[185],"effectiveness;":[187],"(iii)":[188,259],"creators":[191],"create":[193],"counter-misinformation.":[194,213],"This":[195],"explores":[197],"crowd\u2019s":[199],"capabilities":[200],"detection,":[203],"identifies":[204],"prerequisites":[206],"effective":[208],"analyzes":[211],"crowd-generated":[212],"each":[215],"assigned":[216],"conduct":[219],"detailed":[221],"categorize":[224],"specific":[226],"utilization":[227],"crowd.":[230],"Particularly,":[231],"delve":[233],"into":[234],"distinguishing":[236],"individual,":[237],"collaborative,":[238],"machine-assisted":[240],"labeling":[241],"annotators;":[243],"analyzing":[245],"effectiveness":[247],"through":[250],"surveys,":[251],"interviews,":[252],"in-lab":[254],"experiments":[255],"evaluators;":[257],"characterizing":[260],"creation":[261],"patterns":[262],"creator":[264],"profiles":[265],"creators.":[267],"Finally,":[268],"conclude":[270],"outlining":[274],"potential":[275],"avenues":[276],"future":[278],"field.":[282]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-30T17:31:21.811387","created_date":"2025-10-10T00:00:00"}