{"id":"https://openalex.org/W3010633102","doi":"https://doi.org/10.1080/10584609.2020.1723752","title":"In Validations We Trust? The Impact of Imperfect Human Annotations as a Gold Standard on the Quality of Validation of Automated Content Analysis","display_name":"In Validations We Trust? The Impact of Imperfect Human Annotations as a Gold Standard on the Quality of Validation of Automated Content Analysis","publication_year":2020,"publication_date":"2020-03-05","ids":{"openalex":"https://openalex.org/W3010633102","doi":"https://doi.org/10.1080/10584609.2020.1723752","mag":"3010633102"},"language":"en","primary_location":{"id":"doi:10.1080/10584609.2020.1723752","doi":"https://doi.org/10.1080/10584609.2020.1723752","is_oa":true,"landing_page_url":"https://doi.org/10.1080/10584609.2020.1723752","pdf_url":"https://www.tandfonline.com/doi/pdf/10.1080/10584609.2020.1723752?needAccess=true","source":{"id":"https://openalex.org/S27211427","display_name":"Political Communication","issn_l":"1058-4609","issn":["1058-4609","1091-7675"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a","anvur-class-a","anvur-scientific","cwts-core","erih-plus","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547","https://openalex.org/P4310320449"],"host_organization_lineage_names":["Taylor & Francis","Informa"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Political Communication","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://www.tandfonline.com/doi/pdf/10.1080/10584609.2020.1723752?needAccess=true","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015805381","display_name":"Hyunjin Song","orcid":"https://orcid.org/0000-0001-7752-3035","observed_orcids":["https://orcid.org/0000-0001-7752-3035"]},"institutions":[{"id":"https://openalex.org/I129774422","display_name":"University of Vienna","ror":"https://ror.org/03prydq77","country_code":"AT","type":"education","lineage":["https://openalex.org/I129774422"]}],"countries":["AT"],"is_corresponding":true,"raw_author_name":"Hyunjin Song","raw_affiliation_strings":["University of Vienna","Department of Communication, University of Vienna, Vienna, Austria"],"raw_orcid":"https://orcid.org/0000-0001-7752-3035","affiliations":[{"raw_affiliation_string":"University of Vienna","institution_ids":["https://openalex.org/I129774422"]},{"raw_affiliation_string":"Department of Communication, University of Vienna, Vienna, Austria","institution_ids":["https://openalex.org/I129774422"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048083703","display_name":"Petro Tolochko","orcid":"https://orcid.org/0000-0001-5971-8816","observed_orcids":["https://orcid.org/0000-0001-5971-8816"]},"institutions":[{"id":"https://openalex.org/I129774422","display_name":"University of Vienna","ror":"https://ror.org/03prydq77","country_code":"AT","type":"education","lineage":["https://openalex.org/I129774422"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Petro Tolochko","raw_affiliation_strings":["University of Vienna","Department of Political Science, University of Vienna, Vienna, Austria"],"raw_orcid":"https://orcid.org/0000-0001-5971-8816","affiliations":[{"raw_affiliation_string":"University of Vienna","institution_ids":["https://openalex.org/I129774422"]},{"raw_affiliation_string":"Department of Political Science, University of Vienna, Vienna, Austria","institution_ids":["https://openalex.org/I129774422"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037515257","display_name":"Jakob\u2010Moritz Eberl","orcid":"https://orcid.org/0000-0002-5613-760X","observed_orcids":["https://orcid.org/0000-0002-5613-760X"]},"institutions":[{"id":"https://openalex.org/I129774422","display_name":"University of Vienna","ror":"https://ror.org/03prydq77","country_code":"AT","type":"education","lineage":["https://openalex.org/I129774422"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Jakob-Moritz Eberl","raw_affiliation_strings":["University of Vienna","Department of Communication, University of Vienna, Vienna, Austria"],"raw_orcid":"https://orcid.org/0000-0002-5613-760X","affiliations":[{"raw_affiliation_string":"University of Vienna","institution_ids":["https://openalex.org/I129774422"]},{"raw_affiliation_string":"Department of Communication, University of Vienna, Vienna, Austria","institution_ids":["https://openalex.org/I129774422"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053672957","display_name":"Olga Eisele","orcid":"https://orcid.org/0000-0002-6604-3498","observed_orcids":["https://orcid.org/0000-0002-6604-3498"]},"institutions":[{"id":"https://openalex.org/I129774422","display_name":"University of Vienna","ror":"https://ror.org/03prydq77","country_code":"AT","type":"education","lineage":["https://openalex.org/I129774422"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Olga Eisele","raw_affiliation_strings":["University of Vienna","Department of Communication, University of Vienna, Vienna, Austria"],"raw_orcid":"https://orcid.org/0000-0002-6604-3498","affiliations":[{"raw_affiliation_string":"University of Vienna","institution_ids":["https://openalex.org/I129774422"]},{"raw_affiliation_string":"Department of Communication, University of Vienna, Vienna, Austria","institution_ids":["https://openalex.org/I129774422"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003735747","display_name":"Esther Greussing","orcid":"https://orcid.org/0000-0001-8655-5119","observed_orcids":["https://orcid.org/0000-0001-8655-5119"]},"institutions":[{"id":"https://openalex.org/I94509681","display_name":"Technische Universit\u00e4t Braunschweig","ror":"https://ror.org/010nsgg66","country_code":"DE","type":"education","lineage":["https://openalex.org/I94509681"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Esther Greussing","raw_affiliation_strings":["Technische Universit\u00e4t Braunschweig","Department of Communication and Media Sciences, Technische Universit\u00e4t Braunschweig, Braunschweig, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technische Universit\u00e4t Braunschweig","institution_ids":["https://openalex.org/I94509681"]},{"raw_affiliation_string":"Department of Communication and Media Sciences, Technische Universit\u00e4t Braunschweig, Braunschweig, Germany","institution_ids":["https://openalex.org/I94509681"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058683305","display_name":"Tobias Heidenreich","orcid":"https://orcid.org/0000-0001-9070-0550","observed_orcids":["https://orcid.org/0000-0001-9070-0550"]},"institutions":[{"id":"https://openalex.org/I129774422","display_name":"University of Vienna","ror":"https://ror.org/03prydq77","country_code":"AT","type":"education","lineage":["https://openalex.org/I129774422"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Tobias Heidenreich","raw_affiliation_strings":["University of Vienna","Department of Communication, University of Vienna, Vienna, Austria"],"raw_orcid":"https://orcid.org/0000-0001-9070-0550","affiliations":[{"raw_affiliation_string":"University of Vienna","institution_ids":["https://openalex.org/I129774422"]},{"raw_affiliation_string":"Department of Communication, University of Vienna, Vienna, Austria","institution_ids":["https://openalex.org/I129774422"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062349604","display_name":"Fabienne Lind","orcid":"https://orcid.org/0000-0002-4978-9415","observed_orcids":["https://orcid.org/0000-0002-4978-9415"]},"institutions":[{"id":"https://openalex.org/I129774422","display_name":"University of Vienna","ror":"https://ror.org/03prydq77","country_code":"AT","type":"education","lineage":["https://openalex.org/I129774422"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Fabienne Lind","raw_affiliation_strings":["University of Vienna","Department of Communication, University of Vienna, Vienna, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Vienna","institution_ids":["https://openalex.org/I129774422"]},{"raw_affiliation_string":"Department of Communication, University of Vienna, Vienna, Austria","institution_ids":["https://openalex.org/I129774422"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058944754","display_name":"Sebastian Galyga","orcid":"https://orcid.org/0000-0003-2642-0815","observed_orcids":["https://orcid.org/0000-0003-2642-0815"]},"institutions":[{"id":"https://openalex.org/I129774422","display_name":"University of Vienna","ror":"https://ror.org/03prydq77","country_code":"AT","type":"education","lineage":["https://openalex.org/I129774422"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Sebastian Galyga","raw_affiliation_strings":["University of Vienna","Department of Communication, University of Vienna, Vienna, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Vienna","institution_ids":["https://openalex.org/I129774422"]},{"raw_affiliation_string":"Department of Communication, University of Vienna, Vienna, Austria","institution_ids":["https://openalex.org/I129774422"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085826258","display_name":"Hajo G. Boomgaarden","orcid":"https://orcid.org/0000-0002-5260-1284","observed_orcids":["https://orcid.org/0000-0002-5260-1284"]},"institutions":[{"id":"https://openalex.org/I129774422","display_name":"University of Vienna","ror":"https://ror.org/03prydq77","country_code":"AT","type":"education","lineage":["https://openalex.org/I129774422"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Hajo G. Boomgaarden","raw_affiliation_strings":["University of Vienna","Department of Communication, University of Vienna, Vienna, Austria"],"raw_orcid":"https://orcid.org/0000-0002-5260-1284","affiliations":[{"raw_affiliation_string":"University of Vienna","institution_ids":["https://openalex.org/I129774422"]},{"raw_affiliation_string":"Department of Communication, University of Vienna, Vienna, Austria","institution_ids":["https://openalex.org/I129774422"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5015805381"],"corresponding_institution_ids":["https://openalex.org/I129774422"],"apc_list":null,"apc_paid":null,"fwci":43.4227,"has_fulltext":false,"cited_by_count":123,"citation_normalized_percentile":{"value":0.99714104,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":100},"biblio":{"volume":"37","issue":"4","first_page":"550","last_page":"572"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":1.0,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":1.0,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-text-analysis","display_name":"computational text analysis","score":0.7160000205039978},{"id":"https://openalex.org/keywords/validation","display_name":"validation","score":0.4749999940395355},{"id":"https://openalex.org/keywords/gold-standard","display_name":"gold standard","score":0.8519999980926514},{"id":"https://openalex.org/keywords/manual-coding","display_name":"manual coding","score":0.6420000195503235},{"id":"https://openalex.org/keywords/performance-evaluation","display_name":"performance evaluation","score":0.609000027179718},{"id":"https://openalex.org/keywords/text-analysis","display_name":"text analysis","score":0.3330000042915344},{"id":"https://openalex.org/keywords/reporting-practices","display_name":"reporting practices","score":0.4749999940395355},{"id":"https://openalex.org/keywords/monte-carlo-simulation","display_name":"Monte Carlo simulation","score":0.6259999871253967}],"concepts":[{"id":"https://openalex.org/C2780310539","wikidata":"https://www.wikidata.org/wiki/Q12547192","display_name":"Imperfect","level":2,"score":0.7022485733032227},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6234678030014038},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5687063336372375},{"id":"https://openalex.org/C162446236","wikidata":"https://www.wikidata.org/wiki/Q653137","display_name":"Content analysis","level":2,"score":0.539729118347168},{"id":"https://openalex.org/C40993552","wikidata":"https://www.wikidata.org/wiki/Q514654","display_name":"Gold standard (test)","level":2,"score":0.515535831451416},{"id":"https://openalex.org/C2778152352","wikidata":"https://www.wikidata.org/wiki/Q5165061","display_name":"Content (measure theory)","level":2,"score":0.4720737934112549},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.202743798494339},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.16530722379684448},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.11249959468841553},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0968288779258728},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.09048241376876831},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1080/10584609.2020.1723752","doi":"https://doi.org/10.1080/10584609.2020.1723752","is_oa":true,"landing_page_url":"https://doi.org/10.1080/10584609.2020.1723752","pdf_url":"https://www.tandfonline.com/doi/pdf/10.1080/10584609.2020.1723752?needAccess=true","source":{"id":"https://openalex.org/S27211427","display_name":"Political Communication","issn_l":"1058-4609","issn":["1058-4609","1091-7675"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a","anvur-class-a","anvur-scientific","cwts-core","erih-plus","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547","https://openalex.org/P4310320449"],"host_organization_lineage_names":["Taylor & Francis","Informa"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Political Communication","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1080/10584609.2020.1723752","doi":"https://doi.org/10.1080/10584609.2020.1723752","is_oa":true,"landing_page_url":"https://doi.org/10.1080/10584609.2020.1723752","pdf_url":"https://www.tandfonline.com/doi/pdf/10.1080/10584609.2020.1723752?needAccess=true","source":{"id":"https://openalex.org/S27211427","display_name":"Political Communication","issn_l":"1058-4609","issn":["1058-4609","1091-7675"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a","anvur-class-a","anvur-scientific","cwts-core","erih-plus","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547","https://openalex.org/P4310320449"],"host_organization_lineage_names":["Taylor & Francis","Informa"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Political Communication","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4484755181","display_name":"Tango on a Tightrope","funder_award_id":"T 989","funder_id":"https://openalex.org/F4320321181","funder_display_name":"Austrian Science Fund"}],"funders":[{"id":"https://openalex.org/F4320321181","display_name":"Austrian Science Fund","ror":"https://ror.org/013tf3c58"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W3010633102.pdf"},"referenced_works_count":39,"referenced_works":["https://openalex.org/W1968044330","https://openalex.org/W1970381522","https://openalex.org/W1988147352","https://openalex.org/W1993613062","https://openalex.org/W2012378416","https://openalex.org/W2013413196","https://openalex.org/W2016210396","https://openalex.org/W2016862978","https://openalex.org/W2061504941","https://openalex.org/W2072514260","https://openalex.org/W2095655043","https://openalex.org/W2097353125","https://openalex.org/W2109114431","https://openalex.org/W2109528460","https://openalex.org/W2117667023","https://openalex.org/W2141760007","https://openalex.org/W2146179692","https://openalex.org/W2148752429","https://openalex.org/W2153222072","https://openalex.org/W2171060319","https://openalex.org/W2175753910","https://openalex.org/W2188822068","https://openalex.org/W2201853392","https://openalex.org/W2409811014","https://openalex.org/W2521348577","https://openalex.org/W2543606719","https://openalex.org/W2604320488","https://openalex.org/W2611476725","https://openalex.org/W2736885773","https://openalex.org/W2759290493","https://openalex.org/W2790658947","https://openalex.org/W2792932412","https://openalex.org/W2795420284","https://openalex.org/W2802630126","https://openalex.org/W2900377201","https://openalex.org/W2954087776","https://openalex.org/W2956747517","https://openalex.org/W3039884445","https://openalex.org/W4249103792"],"related_works":["https://openalex.org/W2095655043","https://openalex.org/W4381192688","https://openalex.org/W4389094333","https://openalex.org/W4414455916","https://openalex.org/W2187888730","https://openalex.org/W1531343670","https://openalex.org/W4241743845","https://openalex.org/W7100995482","https://openalex.org/W2158880898","https://openalex.org/W4392251150"],"abstract_inverted_index":{"Political":[0],"communication":[1,179],"has":[2],"become":[3],"one":[4],"of":[5,9,14,21,34,51,61,82,95,106,132,142,148,171],"the":[6,12,39,47,75,78,90,93,102,122,140,146,168],"central":[7],"arenas":[8],"innovation":[10],"in":[11,45,74,98,176],"application":[13,170],"automated":[15,43,107,143,183],"analysis":[16,185],"approaches":[17],"to":[18,31,37],"ever-growing":[19],"quantities":[20],"digitized":[22],"texts.":[23],"However,":[24],"although":[25],"researchers":[26],"routinely":[27],"and":[28,77,80,101],"conveniently":[29],"resort":[30],"certain":[32],"forms":[33],"human":[35,96],"coding":[36],"validate":[38],"results":[40,120],"derived":[41],"from":[42,68,121],"procedures,":[44],"practice":[46],"actual":[48],"\u201cquality":[49],"assurance\u201d":[50],"such":[52],"a":[53,128,133,165],"\u201cgold":[54],"standard\u201d":[55],"often":[56],"goes":[57],"unchecked.":[58],"Contemporary":[59],"practices":[60,73],"validation":[62,83,153,174],"via":[63],"manual":[64,99,149,173],"annotations":[65,100,150],"are":[66],"far":[67],"being":[69],"acknowledged":[70],"as":[71,164],"best":[72],"literature,":[76],"reporting":[79],"interpretation":[81],"procedures":[84,108,144],"differ":[85],"greatly.":[86],"We":[87],"systematically":[88],"assess":[89],"connection":[91],"between":[92],"quality":[94,147],"judgment":[97],"relative":[103],"performance":[104,141],"evaluations":[105],"against":[109],"true":[110],"standards":[111],"by":[112],"relying":[113],"on":[114,182],"large-scale":[115],"Monte":[116],"Carlo":[117],"simulations.":[118],"The":[119],"simulations":[123],"con\ufb01rm":[124],"that":[125],"there":[126],"is":[127,154],"substantially":[129],"greater":[130],"risk":[131],"researcher":[134],"reaching":[135],"an":[136],"incorrect":[137],"conclusion":[138],"regarding":[139],"when":[145],"used":[151],"for":[152,167],"not":[155],"properly":[156],"ensured.":[157],"Our":[158],"contribution":[159],"should":[160],"therefore":[161],"be":[162],"regarded":[163],"call":[166],"systematic":[169],"high-quality":[172],"materials":[175],"any":[177],"political":[178],"study,":[180],"drawing":[181],"text":[184],"procedures.":[186]},"counts_by_year":[{"year":2026,"cited_by_count":19},{"year":2025,"cited_by_count":23},{"year":2024,"cited_by_count":20},{"year":2023,"cited_by_count":24},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":14},{"year":2020,"cited_by_count":3}],"updated_date":"2026-10-24T09:50:30.924529","created_date":"2020-03-13T00:00:00"}