{"id":"https://openalex.org/W3024423892","doi":"https://doi.org/10.1145/3397271.3401112","title":"Can The Crowd Identify Misinformation Objectively?","display_name":"Can The Crowd Identify Misinformation Objectively?","publication_year":2020,"publication_date":"2020-07-25","ids":{"openalex":"https://openalex.org/W3024423892","doi":"https://doi.org/10.1145/3397271.3401112","mag":"3024423892"},"language":"en","primary_location":{"id":"doi:10.1145/3397271.3401112","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3397271.3401112","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/abs/2005.06915","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050287516","display_name":"Kevin Roitero","orcid":"https://orcid.org/0000-0002-9191-3280"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Kevin Roitero","raw_affiliation_strings":["University of Udine, Udine, Italy","University of Udine, Udine (Italy)#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Udine, Udine, Italy","institution_ids":["https://openalex.org/I129043915"]},{"raw_affiliation_string":"University of Udine, Udine (Italy)#TAB#","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028918939","display_name":"Michael Soprano","orcid":"https://orcid.org/0000-0002-7337-7592"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Michael Soprano","raw_affiliation_strings":["University of Udine, Udine, Italy","University of Udine, Udine (Italy)#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Udine, Udine, Italy","institution_ids":["https://openalex.org/I129043915"]},{"raw_affiliation_string":"University of Udine, Udine (Italy)#TAB#","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050249085","display_name":"Shaoyang Fan","orcid":"https://orcid.org/0000-0002-4716-4497"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Shaoyang Fan","raw_affiliation_strings":["University of Queensland, Brisbane, Australia","University of Queensland Brisbane Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Queensland, Brisbane, Australia","institution_ids":["https://openalex.org/I165143802"]},{"raw_affiliation_string":"University of Queensland Brisbane Australia","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083881309","display_name":"Damiano Spina","orcid":"https://orcid.org/0000-0001-9913-433X"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Damiano Spina","raw_affiliation_strings":["RMIT University, Melbourne, Australia","(RMIT University, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]},{"raw_affiliation_string":"(RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005575421","display_name":"Stefano Mizzaro","orcid":"https://orcid.org/0000-0002-2852-168X"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Stefano Mizzaro","raw_affiliation_strings":["University of Udine, Udine, Italy","University of Udine, Udine (Italy)#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Udine, Udine, Italy","institution_ids":["https://openalex.org/I129043915"]},{"raw_affiliation_string":"University of Udine, Udine (Italy)#TAB#","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052565959","display_name":"Gianluca Demartini","orcid":"https://orcid.org/0000-0002-7311-3693"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Gianluca Demartini","raw_affiliation_strings":["University of Queensland, Brisbane, Australia","University of Queensland Brisbane Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Queensland, Brisbane, Australia","institution_ids":["https://openalex.org/I165143802"]},{"raw_affiliation_string":"University of Queensland Brisbane Australia","institution_ids":["https://openalex.org/I165143802"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"439","last_page":"448"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":1.0,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":1.0,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":0.9857000112533569,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9830999970436096,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9318437576293945},{"id":"https://openalex.org/keywords/misinformation","display_name":"Misinformation","score":0.9017086029052734},{"id":"https://openalex.org/keywords/credibility","display_name":"Credibility","score":0.7452390789985657},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7253566980361938},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6122279167175293},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5751678943634033},{"id":"https://openalex.org/keywords/granularity","display_name":"Granularity","score":0.4544166028499603},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4422796070575714},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3782305419445038},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.20218664407730103},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.17536312341690063},{"id":"https://openalex.org/keywords/power","display_name":"Power (physics)","score":0.08726590871810913}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9318437576293945},{"id":"https://openalex.org/C2776990098","wikidata":"https://www.wikidata.org/wiki/Q13579947","display_name":"Misinformation","level":2,"score":0.9017086029052734},{"id":"https://openalex.org/C2780224610","wikidata":"https://www.wikidata.org/wiki/Q1530061","display_name":"Credibility","level":2,"score":0.7452390789985657},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7253566980361938},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6122279167175293},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5751678943634033},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.4544166028499603},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4422796070575714},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3782305419445038},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.20218664407730103},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.17536312341690063},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.08726590871810913},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3397271.3401112","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3397271.3401112","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"mag:3024423892","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2005.06915","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:3024423892","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2005.06915","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.6800000071525574}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W136732505","https://openalex.org/W1873424132","https://openalex.org/W1992549066","https://openalex.org/W2014538668","https://openalex.org/W2021735564","https://openalex.org/W2041282815","https://openalex.org/W2043845271","https://openalex.org/W2051442094","https://openalex.org/W2101217916","https://openalex.org/W2103542834","https://openalex.org/W2110785023","https://openalex.org/W2118852500","https://openalex.org/W2140448605","https://openalex.org/W2152314154","https://openalex.org/W2161080627","https://openalex.org/W2164733043","https://openalex.org/W2251214593","https://openalex.org/W2294699749","https://openalex.org/W2410465342","https://openalex.org/W2413573154","https://openalex.org/W2480455979","https://openalex.org/W2569307494","https://openalex.org/W2591615201","https://openalex.org/W2756774204","https://openalex.org/W2760642188","https://openalex.org/W2763572884","https://openalex.org/W2765320432","https://openalex.org/W2766411764","https://openalex.org/W2772256541","https://openalex.org/W2783668259","https://openalex.org/W2790166049","https://openalex.org/W2794432940","https://openalex.org/W2798782484","https://openalex.org/W2798953174","https://openalex.org/W2809476703","https://openalex.org/W2887356313","https://openalex.org/W2901185870","https://openalex.org/W2912571444","https://openalex.org/W2913081045","https://openalex.org/W2941486823","https://openalex.org/W2947718494","https://openalex.org/W2955681503","https://openalex.org/W2963416784","https://openalex.org/W2963709474","https://openalex.org/W2968104057","https://openalex.org/W2970865836","https://openalex.org/W2980516042","https://openalex.org/W2980649541","https://openalex.org/W2986224017","https://openalex.org/W2991361764","https://openalex.org/W2997622041","https://openalex.org/W3016244823","https://openalex.org/W3101890897","https://openalex.org/W3121274724","https://openalex.org/W3122482117","https://openalex.org/W3122883818","https://openalex.org/W4229508619"],"related_works":["https://openalex.org/W10626366","https://openalex.org/W8975994","https://openalex.org/W3692153","https://openalex.org/W4512113","https://openalex.org/W181698","https://openalex.org/W8546146","https://openalex.org/W3140752","https://openalex.org/W4409431","https://openalex.org/W8791693","https://openalex.org/W9937398"],"abstract_inverted_index":{"Truthfulness":[0],"judgments":[1,30,118],"are":[2,14,31],"a":[3,51],"fundamental":[4],"step":[5],"in":[6],"the":[7,65,76,96,132,136,140,147,150,154],"process":[8],"of":[9,61,78,98,108,139,149],"fighting":[10],"misinformation,":[11],"as":[12],"they":[13],"crucial":[15],"to":[16,64,74,81],"train":[17],"and":[18,25,54,80,114,135,142],"evaluate":[19],"classifiers":[20],"that":[21],"automatically":[22],"distinguish":[23],"true":[24],"false":[26],"statements.":[27,45],"Usually":[28],"such":[29],"made":[32],"by":[33,153],"experts,":[34],"like":[35],"journalists":[36],"for":[37,43,86],"political":[38,133],"statements":[39],"or":[40],"medical":[41,44],"doctors":[42],"In":[46],"this":[47,92],"paper,":[48],"we":[49,94,105,115],"follow":[50],"different":[52],"approach":[53],"rely":[55],"on":[56,103,123,146],"(non-expert)":[57],"crowd":[58,120],"workers.":[59],"This":[60],"course":[62],"leads":[63],"following":[66],"research":[67],"question:":[68],"Can":[69],"crowdsourcing":[70],"be":[71],"reliably":[72],"used":[73],"assess":[75],"truthfulness":[77,109],"information":[79,87],"create":[82],"large-scale":[83],"labeled":[84],"collections":[85],"credibility":[88],"systems?":[89],"To":[90],"address":[91],"issue,":[93],"present":[95],"results":[97],"an":[99],"extensive":[100],"study":[101],"based":[102],"crowdsourcing:":[104],"collect":[106],"thousands":[107],"assessments":[110],"over":[111],"two":[112],"datasets,":[113],"compare":[116],"expert":[117],"with":[119,125],"judgments,":[121],"expressed":[122],"scales":[124],"various":[126],"granularity":[127],"levels.":[128],"We":[129],"also":[130],"measure":[131],"bias":[134],"cognitive":[137],"background":[138],"workers,":[141],"quantify":[143],"their":[144],"effect":[145],"reliability":[148],"data":[151],"provided":[152],"crowd.":[155]},"counts_by_year":[{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}