{"id":"https://openalex.org/W3080732897","doi":"https://doi.org/10.1145/3415164","title":"Investigating Differences in Crowdsourced News Credibility Assessment","display_name":"Investigating Differences in Crowdsourced News Credibility Assessment","publication_year":2020,"publication_date":"2020-10-14","ids":{"openalex":"https://openalex.org/W3080732897","doi":"https://doi.org/10.1145/3415164","mag":"3080732897"},"language":"en","primary_location":{"id":"doi:10.1145/3415164","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3415164","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3415164","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3415164","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039539331","display_name":"Md Momen Bhuiyan","orcid":"https://orcid.org/0000-0003-3408-9122"},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Md Momen Bhuiyan","raw_affiliation_strings":["Virginia Tech, Blacksburg, VA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Virginia Tech, Blacksburg, VA, USA","institution_ids":["https://openalex.org/I859038795"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037569091","display_name":"Amy X. Zhang","orcid":"https://orcid.org/0000-0001-9462-9835"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amy X. Zhang","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016868030","display_name":"Connie Moon Sehat","orcid":"https://orcid.org/0000-0002-4803-0730"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Connie Moon Sehat","raw_affiliation_strings":["Hacks/Hackers, Atlanta, GA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hacks/Hackers, Atlanta, GA, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062498796","display_name":"Tanushree Mitra","orcid":"https://orcid.org/0000-0002-9507-6192"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tanushree Mitra","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5039539331"],"corresponding_institution_ids":["https://openalex.org/I859038795"],"apc_list":null,"apc_paid":null,"fwci":12.1117,"has_fulltext":true,"cited_by_count":62,"citation_normalized_percentile":{"value":0.98476474,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"4","issue":"CSCW2","first_page":"1","last_page":"26"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":1.0,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":1.0,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10557","display_name":"Social Media and Politics","score":0.9930999875068665,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12592","display_name":"Opinion Dynamics and Social Influence","score":0.9803000092506409,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.864102840423584},{"id":"https://openalex.org/keywords/misinformation","display_name":"Misinformation","score":0.8569788336753845},{"id":"https://openalex.org/keywords/credibility","display_name":"Credibility","score":0.8076012134552002},{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.7086021900177002},{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.47388380765914917},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4462594985961914},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.436744749546051},{"id":"https://openalex.org/keywords/journalism","display_name":"Journalism","score":0.42623552680015564},{"id":"https://openalex.org/keywords/public-opinion","display_name":"Public opinion","score":0.4147742986679077},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3871118426322937},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.36229318380355835},{"id":"https://openalex.org/keywords/public-relations","display_name":"Public relations","score":0.3449096083641052},{"id":"https://openalex.org/keywords/politics","display_name":"Politics","score":0.27824127674102783},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.27772313356399536},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.18708309531211853},{"id":"https://openalex.org/keywords/advertising","display_name":"Advertising","score":0.154129296541214},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.13389676809310913},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.09198465943336487}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.864102840423584},{"id":"https://openalex.org/C2776990098","wikidata":"https://www.wikidata.org/wiki/Q13579947","display_name":"Misinformation","level":2,"score":0.8569788336753845},{"id":"https://openalex.org/C2780224610","wikidata":"https://www.wikidata.org/wiki/Q1530061","display_name":"Credibility","level":2,"score":0.8076012134552002},{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.7086021900177002},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.47388380765914917},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4462594985961914},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.436744749546051},{"id":"https://openalex.org/C119513131","wikidata":"https://www.wikidata.org/wiki/Q11030","display_name":"Journalism","level":2,"score":0.42623552680015564},{"id":"https://openalex.org/C134698397","wikidata":"https://www.wikidata.org/wiki/Q17946","display_name":"Public opinion","level":3,"score":0.4147742986679077},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3871118426322937},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.36229318380355835},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.3449096083641052},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.27824127674102783},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.27772313356399536},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.18708309531211853},{"id":"https://openalex.org/C112698675","wikidata":"https://www.wikidata.org/wiki/Q37038","display_name":"Advertising","level":1,"score":0.154129296541214},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.13389676809310913},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.09198465943336487},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3415164","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3415164","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3415164","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2008.09533","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2008.09533","pdf_url":"https://arxiv.org/pdf/2008.09533","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3415164","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3415164","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3415164","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.8199999928474426,"id":"https://metadata.un.org/sdg/13","display_name":"Climate action"}],"awards":[{"id":"https://openalex.org/G2547889013","display_name":null,"funder_award_id":"1755547","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3080732897.pdf","grobid_xml":"https://content.openalex.org/works/W3080732897.grobid-xml"},"referenced_works_count":66,"referenced_works":["https://openalex.org/W106703717","https://openalex.org/W185484974","https://openalex.org/W621249151","https://openalex.org/W1249991712","https://openalex.org/W1503758377","https://openalex.org/W1539896512","https://openalex.org/W1605255363","https://openalex.org/W1915697785","https://openalex.org/W1930551088","https://openalex.org/W1954347551","https://openalex.org/W1962580118","https://openalex.org/W1982099510","https://openalex.org/W1982149803","https://openalex.org/W1984640266","https://openalex.org/W1984763500","https://openalex.org/W1996118086","https://openalex.org/W2011109956","https://openalex.org/W2011399107","https://openalex.org/W2013974666","https://openalex.org/W2042114350","https://openalex.org/W2058179030","https://openalex.org/W2062117700","https://openalex.org/W2073375484","https://openalex.org/W2094747961","https://openalex.org/W2095942159","https://openalex.org/W2110151287","https://openalex.org/W2120220258","https://openalex.org/W2121155993","https://openalex.org/W2130412759","https://openalex.org/W2145403478","https://openalex.org/W2155479778","https://openalex.org/W2162239230","https://openalex.org/W2278631569","https://openalex.org/W2288053829","https://openalex.org/W2294096789","https://openalex.org/W2327590731","https://openalex.org/W2416868893","https://openalex.org/W2468245033","https://openalex.org/W2476065884","https://openalex.org/W2604264634","https://openalex.org/W2611363225","https://openalex.org/W2745461309","https://openalex.org/W2770039499","https://openalex.org/W2798818974","https://openalex.org/W2910172717","https://openalex.org/W2910632137","https://openalex.org/W2911111957","https://openalex.org/W2912659123","https://openalex.org/W2913081045","https://openalex.org/W2913649461","https://openalex.org/W2915714879","https://openalex.org/W2944938587","https://openalex.org/W2945941328","https://openalex.org/W2956077981","https://openalex.org/W2968618281","https://openalex.org/W2972200238","https://openalex.org/W3013470162","https://openalex.org/W3032049092","https://openalex.org/W3099727075","https://openalex.org/W3104758113","https://openalex.org/W3122883818","https://openalex.org/W3125618971","https://openalex.org/W4206273048","https://openalex.org/W4230939871","https://openalex.org/W4244762921","https://openalex.org/W4256057297"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W3197131596","https://openalex.org/W1503094549","https://openalex.org/W4390616380","https://openalex.org/W4388666321","https://openalex.org/W4205914924","https://openalex.org/W2337920774","https://openalex.org/W4283459170"],"abstract_inverted_index":{"Misinformation":[0],"about":[1],"critical":[2],"issues":[3],"such":[4,130,152],"as":[5,31,87,131,137,139,153],"climate":[6,102],"change":[7],"and":[8,17,62,83,91,112,134,159,226],"vaccine":[9],"safety":[10],"is":[11,148],"oftentimes":[12],"amplified":[13],"on":[14],"online":[15],"social":[16],"search":[18],"platforms.":[19],"The":[20],"crowdsourcing":[21],"of":[22,43,72,96,127,142,156,161,228],"content":[23],"credibility":[24,53,75],"assessment":[25],"by":[26,37,55,198],"laypeople":[27],"has":[28],"been":[29],"proposed":[30],"one":[32],"strategy":[33],"to":[34,39,59,101,124,150,172,185,194,203,215,223],"combat":[35],"misinformation":[36],"attempting":[38],"replicate":[40],"the":[41,116,125,128,140,143,146,154,157,162,196],"assessments":[42,54,76,170],"experts":[44,58,180],"at":[45],"scale.":[46],"In":[47],"this":[48],"work,":[49],"we":[50,118,165,210],"investigate":[51],"news":[52,98],"crowds":[56,225],"versus":[57,178],"understand":[60],"when":[61],"how":[63],"ratings":[64],"between":[65,109,168],"them":[66],"differ.":[67],"We":[68],"gather":[69],"a":[70,93,104,192],"dataset":[71],"over":[73],"4,000":[74],"taken":[77],"from":[78],"2":[79,88],"crowd":[80,147,186,200,218],"groups---journalism":[81],"students":[82],"Upwork":[84],"workers---as":[85],"well":[86,138],"expert":[89,113,169,174,205],"groups---journalists":[90],"scientists---on":[92],"varied":[94],"set":[95],"50":[97],"articles":[99],"related":[100],"science,":[103],"topic":[105],"with":[106],"widespread":[107],"disconnect":[108],"public":[110],"opinion":[111],"consensus.":[114],"Examining":[115],"ratings,":[117],"find":[119,166],"differences":[120,167],"in":[121],"performance":[122],"due":[123,171],"makeup":[126],"crowd,":[129],"rater":[132],"demographics":[133],"political":[135],"leaning,":[136],"scope":[141],"tasks":[144,201],"that":[145,176,182,189,220],"assigned":[149],"rate,":[151],"genre":[155],"article":[158],"partisanship":[160],"publication.":[163],"Finally,":[164],"differing":[173],"criteria":[175],"journalism":[177],"science":[179],"use---differences":[181],"may":[183],"contribute":[184],"discrepancies,":[187],"but":[188],"also":[190],"suggest":[191],"way":[193],"reduce":[195],"gap":[197],"designing":[199],"tailored":[202,222],"specific":[204,224],"criteria.":[206],"From":[207],"these":[208],"findings,":[209],"outline":[211],"future":[212],"research":[213],"directions":[214],"better":[216],"design":[217],"processes":[219],"are":[221],"types":[227],"content.":[229]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":12},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":1}],"updated_date":"2026-05-04T08:30:34.212998","created_date":"2020-09-01T00:00:00"}