{"id":"https://openalex.org/W4392156304","doi":"https://doi.org/10.3390/bdcc8030024","title":"Experimental Evaluation: Can Humans Recognise Social Media Bots?","display_name":"Experimental Evaluation: Can Humans Recognise Social Media Bots?","publication_year":2024,"publication_date":"2024-02-26","ids":{"openalex":"https://openalex.org/W4392156304","doi":"https://doi.org/10.3390/bdcc8030024"},"language":"en","primary_location":{"id":"doi:10.3390/bdcc8030024","is_oa":true,"landing_page_url":"https://doi.org/10.3390/bdcc8030024","pdf_url":"https://www.mdpi.com/2504-2289/8/3/24/pdf?version=1708943848","source":{"id":"https://openalex.org/S4210238752","display_name":"Big Data and Cognitive Computing","issn_l":"2504-2289","issn":["2504-2289"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Big Data and Cognitive Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2504-2289/8/3/24/pdf?version=1708943848","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065180547","display_name":"Maxim Kolomeets","orcid":"https://orcid.org/0000-0002-7873-2733"},"institutions":[{"id":"https://openalex.org/I1313323035","display_name":"Russian Academy of Sciences","ror":"https://ror.org/05qrfxd25","country_code":"RU","type":"funder","lineage":["https://openalex.org/I1313323035"]},{"id":"https://openalex.org/I84884186","display_name":"Newcastle University","ror":"https://ror.org/01kj2bm70","country_code":"GB","type":"education","lineage":["https://openalex.org/I84884186"]}],"countries":["GB","RU"],"is_corresponding":false,"raw_author_name":"Maxim Kolomeets","raw_affiliation_strings":["School of Computing, Newcastle University, Newcastle upon Tyne NE4 5TG, UK","St. Petersburg Federal Research Center of the Russian Academy of Sciences, 14th Line of V.O. 39, St. Petersburg 199178, Russia"],"affiliations":[{"raw_affiliation_string":"School of Computing, Newcastle University, Newcastle upon Tyne NE4 5TG, UK","institution_ids":["https://openalex.org/I84884186"]},{"raw_affiliation_string":"St. Petersburg Federal Research Center of the Russian Academy of Sciences, 14th Line of V.O. 39, St. Petersburg 199178, Russia","institution_ids":["https://openalex.org/I1313323035"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054088952","display_name":"Olga Tushkanova","orcid":"https://orcid.org/0000-0001-8394-0783"},"institutions":[{"id":"https://openalex.org/I1313323035","display_name":"Russian Academy of Sciences","ror":"https://ror.org/05qrfxd25","country_code":"RU","type":"funder","lineage":["https://openalex.org/I1313323035"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Olga Tushkanova","raw_affiliation_strings":["St. Petersburg Federal Research Center of the Russian Academy of Sciences, 14th Line of V.O. 39, St. Petersburg 199178, Russia"],"affiliations":[{"raw_affiliation_string":"St. Petersburg Federal Research Center of the Russian Academy of Sciences, 14th Line of V.O. 39, St. Petersburg 199178, Russia","institution_ids":["https://openalex.org/I1313323035"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002944317","display_name":"Vasily Desnitsky","orcid":"https://orcid.org/0000-0002-3748-5414"},"institutions":[{"id":"https://openalex.org/I1313323035","display_name":"Russian Academy of Sciences","ror":"https://ror.org/05qrfxd25","country_code":"RU","type":"funder","lineage":["https://openalex.org/I1313323035"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Vasily Desnitsky","raw_affiliation_strings":["St. Petersburg Federal Research Center of the Russian Academy of Sciences, 14th Line of V.O. 39, St. Petersburg 199178, Russia"],"affiliations":[{"raw_affiliation_string":"St. Petersburg Federal Research Center of the Russian Academy of Sciences, 14th Line of V.O. 39, St. Petersburg 199178, Russia","institution_ids":["https://openalex.org/I1313323035"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072630025","display_name":"Lidia Vitkova","orcid":"https://orcid.org/0000-0002-4945-6151"},"institutions":[{"id":"https://openalex.org/I1313323035","display_name":"Russian Academy of Sciences","ror":"https://ror.org/05qrfxd25","country_code":"RU","type":"funder","lineage":["https://openalex.org/I1313323035"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Lidia Vitkova","raw_affiliation_strings":["St. Petersburg Federal Research Center of the Russian Academy of Sciences, 14th Line of V.O. 39, St. Petersburg 199178, Russia"],"affiliations":[{"raw_affiliation_string":"St. Petersburg Federal Research Center of the Russian Academy of Sciences, 14th Line of V.O. 39, St. Petersburg 199178, Russia","institution_ids":["https://openalex.org/I1313323035"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001301716","display_name":"Andrey Chechulin","orcid":"https://orcid.org/0000-0001-7056-6972"},"institutions":[{"id":"https://openalex.org/I1313323035","display_name":"Russian Academy of Sciences","ror":"https://ror.org/05qrfxd25","country_code":"RU","type":"funder","lineage":["https://openalex.org/I1313323035"]}],"countries":["RU"],"is_corresponding":true,"raw_author_name":"Andrey Chechulin","raw_affiliation_strings":["St. Petersburg Federal Research Center of the Russian Academy of Sciences, 14th Line of V.O. 39, St. Petersburg 199178, Russia"],"affiliations":[{"raw_affiliation_string":"St. Petersburg Federal Research Center of the Russian Academy of Sciences, 14th Line of V.O. 39, St. Petersburg 199178, Russia","institution_ids":["https://openalex.org/I1313323035"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5001301716"],"corresponding_institution_ids":["https://openalex.org/I1313323035"],"apc_list":{"value":1400,"currency":"CHF","value_usd":1515},"apc_paid":{"value":1400,"currency":"CHF","value_usd":1515},"fwci":4.0592,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":{"value":0.93863702,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"8","issue":"3","first_page":"24","last_page":"24"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/social-media","display_name":"Social media","score":0.6639745235443115},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.47242796421051025},{"id":"https://openalex.org/keywords/internet-privacy","display_name":"Internet privacy","score":0.450337678194046},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.3389746844768524},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.26355811953544617}],"concepts":[{"id":"https://openalex.org/C518677369","wikidata":"https://www.wikidata.org/wiki/Q202833","display_name":"Social media","level":2,"score":0.6639745235443115},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.47242796421051025},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.450337678194046},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3389746844768524},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.26355811953544617}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3390/bdcc8030024","is_oa":true,"landing_page_url":"https://doi.org/10.3390/bdcc8030024","pdf_url":"https://www.mdpi.com/2504-2289/8/3/24/pdf?version=1708943848","source":{"id":"https://openalex.org/S4210238752","display_name":"Big Data and Cognitive Computing","issn_l":"2504-2289","issn":["2504-2289"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Big Data and Cognitive Computing","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:17f56c11cb304787bb919a14ec976c22","is_oa":true,"landing_page_url":"https://doaj.org/article/17f56c11cb304787bb919a14ec976c22","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Big Data and Cognitive Computing, Vol 8, Iss 3, p 24 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/bdcc8030024","is_oa":true,"landing_page_url":"https://doi.org/10.3390/bdcc8030024","pdf_url":"https://www.mdpi.com/2504-2289/8/3/24/pdf?version=1708943848","source":{"id":"https://openalex.org/S4210238752","display_name":"Big Data and Cognitive Computing","issn_l":"2504-2289","issn":["2504-2289"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Big Data and Cognitive Computing","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4392156304.pdf"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W1837843568","https://openalex.org/W1849719402","https://openalex.org/W1998871422","https://openalex.org/W2117859196","https://openalex.org/W2278635123","https://openalex.org/W2610773131","https://openalex.org/W2765787211","https://openalex.org/W2794977433","https://openalex.org/W2916302242","https://openalex.org/W2937089894","https://openalex.org/W2979163289","https://openalex.org/W3019320019","https://openalex.org/W3041367927","https://openalex.org/W3104378754","https://openalex.org/W3125182500","https://openalex.org/W3125928061","https://openalex.org/W3164090709","https://openalex.org/W3185802882","https://openalex.org/W4231258178","https://openalex.org/W4240802145","https://openalex.org/W4289253852","https://openalex.org/W4310145577","https://openalex.org/W4312579173","https://openalex.org/W4318700067","https://openalex.org/W4321488594","https://openalex.org/W4322503172","https://openalex.org/W4367047167","https://openalex.org/W4389523769","https://openalex.org/W6768834965","https://openalex.org/W6799507664"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2478288626","https://openalex.org/W4391913857","https://openalex.org/W2350741829","https://openalex.org/W2530322880"],"abstract_inverted_index":{"This":[0],"paper":[1,216,281],"aims":[2],"to":[3,42,64,204,232,261],"test":[4,72],"the":[5,8,88,95,99,109,113,181,196,219,234,237,241,262],"hypothesis":[6],"that":[7,30,156,190],"quality":[9],"of":[10,117,142,183,199,243],"social":[11,90,144],"media":[12],"bot":[13,101,114,187,210,248],"detection":[14,115,129,134,139,149,188,213,249],"systems":[15,189,201,250],"based":[16],"on":[17,87,108,209,282],"supervised":[18,65],"machine":[19,227,274],"learning":[20,228,275],"may":[21],"not":[22,60,169],"be":[23],"as":[24,26,131,194],"accurate":[25],"researchers":[27],"claim,":[28],"given":[29],"bots":[31,86,162],"have":[32],"become":[33],"increasingly":[34],"sophisticated,":[35],"making":[36],"it":[37],"difficult":[38],"for":[39,123,270],"human":[40,57,245],"annotators":[41,118],"detect":[43,160,170],"them":[44],"better":[45],"than":[46],"random":[47,136],"selection.":[48],"As":[49],"a":[50,53,105,143],"result,":[51],"obtaining":[52],"ground-truth":[54],"dataset":[55],"with":[56,83,98],"annotation":[58,69],"is":[59],"possible,":[61],"which":[62],"leads":[63],"machine-learning":[66],"models":[67],"inheriting":[68],"errors.":[70],"To":[71],"this":[73,280],"hypothesis,":[74],"we":[75,111,239],"conducted":[76],"an":[77],"experiment":[78,220,265],"where":[79],"humans":[80,157],"were":[81],"tasked":[82],"recognizing":[84],"malicious":[85],"VKontakte":[89],"network.":[91],"We":[92,256],"then":[93],"compared":[94],"\u201chuman\u201d":[96],"answers":[97],"\u201cground-truth\u201d":[100],"labels":[102],"(\u2018a":[103],"bot\u2019/\u2018not":[104],"bot\u2019).":[106],"Based":[107],"experiment,":[110],"evaluated":[112],"efficiency":[116],"in":[119,127,163,247,279],"three":[120,165],"scenarios":[121,166],"typical":[122],"cybersecurity":[124,254],"but":[125,167],"differing":[126],"their":[128],"difficulty":[130],"follows:":[132,195],"(1)":[133],"among":[135,140,150],"accounts,":[137],"(2)":[138],"accounts":[141],"network":[145],"\u2018community\u2019,":[146],"and":[147,185,206,212,226,251,267,273],"(3)":[148],"verified":[151],"accounts.":[152],"The":[153,177,215],"study":[154,178],"showed":[155],"could":[158,168],"only":[159],"simple":[161],"all":[164],"more":[171],"sophisticated":[172],"ones":[173],"(p-value":[174],"=":[175],"0.05).":[176],"also":[179,217,257],"evaluates":[180],"limits":[182],"hypothetical":[184],"existing":[186],"leverage":[191],"non-expert-labelled":[192],"datasets":[193,263],"balanced":[197],"accuracy":[198,229,276],"such":[200],"can":[202],"drop":[203],"0.5":[205],"lower,":[207],"depending":[208],"complexity":[211],"scenario.":[214],"describes":[218],"design,":[221],"collected":[222],"datasets,":[223],"statistical":[224,272],"evaluation,":[225],"measures":[230],"applied":[231],"support":[233],"results.":[235],"In":[236],"discussion,":[238],"raise":[240],"question":[242],"using":[244],"labelling":[246],"its":[252],"potential":[253],"issues.":[255],"provide":[258],"open":[259],"access":[260],"used,":[264],"results,":[266],"software":[268],"code":[269],"evaluating":[271],"metrics":[277],"used":[278],"GitHub.":[283]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2}],"updated_date":"2025-12-21T23:12:01.093139","created_date":"2025-10-10T00:00:00"}