{"id":"https://openalex.org/W1972622142","doi":"https://doi.org/10.1371/journal.pone.0061401","title":"Substantial Agreement of Referee Recommendations at a General Medical Journal \u2013 A Peer Review Evaluation at Deutsches \u00c4rzteblatt International","display_name":"Substantial Agreement of Referee Recommendations at a General Medical Journal \u2013 A Peer Review Evaluation at Deutsches \u00c4rzteblatt International","publication_year":2013,"publication_date":"2013-05-02","ids":{"openalex":"https://openalex.org/W1972622142","doi":"https://doi.org/10.1371/journal.pone.0061401","mag":"1972622142","pmid":"https://pubmed.ncbi.nlm.nih.gov/23658692","pmcid":"https://www.ncbi.nlm.nih.gov/pmc/articles/3642182"},"language":"en","primary_location":{"id":"doi:10.1371/journal.pone.0061401","doi":"https://doi.org/10.1371/journal.pone.0061401","is_oa":true,"landing_page_url":"https://doi.org/10.1371/journal.pone.0061401","pdf_url":"https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0061401&type=printable","source":{"id":"https://openalex.org/S202381698","display_name":"PLoS ONE","issn_l":"1932-6203","issn":["1932-6203"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-c","doyens","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315706","host_organization_name":"Public Library of Science","host_organization_lineage":["https://openalex.org/P4310315706"],"host_organization_lineage_names":["Public Library of Science"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PLoS ONE","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0061401&type=printable","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044042153","display_name":"Christopher Baethge","orcid":"https://orcid.org/0000-0001-6246-3674","observed_orcids":["https://orcid.org/0000-0001-6246-3674"]},"institutions":[{"id":"https://openalex.org/I180923762","display_name":"University of Cologne","ror":"https://ror.org/00rcxh774","country_code":"DE","type":"education","lineage":["https://openalex.org/I180923762"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Christopher Baethge","raw_affiliation_strings":["Deutsches \u00c4rzteblatt International, Editorial Offices, Cologne, Germany. baethge@aerzteblatt.de","Department of Psychiatry and Psychotherapy, University of Cologne Medical School, Cologne, Germany","Deutsches \u00c4rzteblatt International, Editorial Offices, Cologne, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Deutsches \u00c4rzteblatt International, Editorial Offices, Cologne, Germany. baethge@aerzteblatt.de","institution_ids":[]},{"raw_affiliation_string":"Department of Psychiatry and Psychotherapy, University of Cologne Medical School, Cologne, Germany","institution_ids":["https://openalex.org/I180923762"]},{"raw_affiliation_string":"Deutsches \u00c4rzteblatt International, Editorial Offices, Cologne, Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057543920","display_name":"Jeremy G. Franklin","orcid":"https://orcid.org/0000-0003-1536-0925","observed_orcids":["https://orcid.org/0000-0003-1536-0925"]},"institutions":[{"id":"https://openalex.org/I180923762","display_name":"University of Cologne","ror":"https://ror.org/00rcxh774","country_code":"DE","type":"education","lineage":["https://openalex.org/I180923762"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Jeremy Franklin","raw_affiliation_strings":["Institute of Medical Statistics, Informatics, and Epidemiology, University of Cologne Medical School, Cologne, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Medical Statistics, Informatics, and Epidemiology, University of Cologne Medical School, Cologne, Germany","institution_ids":["https://openalex.org/I180923762"]}]},{"author_position":"last","author":{"id":null,"display_name":"Stephan Mertens","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stephan Mertens","raw_affiliation_strings":["Deutsches \u00c4rzteblatt International, Editorial Offices, Cologne, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Deutsches \u00c4rzteblatt International, Editorial Offices, Cologne, Germany","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5044042153"],"corresponding_institution_ids":["https://openalex.org/I180923762"],"apc_list":{"value":2382,"currency":"USD","value_usd":2382},"apc_paid":{"value":2382,"currency":"USD","value_usd":2382},"fwci":36.0731,"has_fulltext":true,"cited_by_count":32,"citation_normalized_percentile":{"value":0.99593155,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":99},"biblio":{"volume":"8","issue":"5","first_page":"e61401","last_page":"e61401"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13516","display_name":"Publishing and Scholarly Communication","score":0.6923999786376953,"subfield":{"id":"https://openalex.org/subfields/1207","display_name":"History and Philosophy of Science"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13516","display_name":"Publishing and Scholarly Communication","score":0.6923999786376953,"subfield":{"id":"https://openalex.org/subfields/1207","display_name":"History and Philosophy of Science"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12648","display_name":"Academic Writing and Publishing","score":0.08910000324249268,"subfield":{"id":"https://openalex.org/subfields/1207","display_name":"History and Philosophy of Science"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.04320000112056732,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/peer-review","display_name":"peer review","score":0.9810000061988831},{"id":"https://openalex.org/keywords/reviewer-agreement","display_name":"reviewer agreement","score":0.8970000147819519},{"id":"https://openalex.org/keywords/editorial-decision-making","display_name":"editorial decision-making","score":0.8399999737739563},{"id":"https://openalex.org/keywords/kappa-statistic","display_name":"kappa statistic","score":0.5730000138282776},{"id":"https://openalex.org/keywords/gwet-s-ac1","display_name":"Gwet's AC1","score":0.8859999775886536},{"id":"https://openalex.org/keywords/citation-impact","display_name":"citation impact","score":0.5600000023841858},{"id":"https://openalex.org/keywords/general-medical-journals","display_name":"general medical journals","score":0.5929999947547913}],"concepts":[{"id":"https://openalex.org/C160798450","wikidata":"https://www.wikidata.org/wiki/Q4230870","display_name":"Concordance","level":2,"score":0.82010418176651},{"id":"https://openalex.org/C163864269","wikidata":"https://www.wikidata.org/wiki/Q1107106","display_name":"Cohen's kappa","level":2,"score":0.6940443515777588},{"id":"https://openalex.org/C138368954","wikidata":"https://www.wikidata.org/wiki/Q215028","display_name":"Peer review","level":2,"score":0.6462723016738892},{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.6385480761528015},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.47793084383010864},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.4772741198539734},{"id":"https://openalex.org/C95190672","wikidata":"https://www.wikidata.org/wiki/Q815382","display_name":"Meta-analysis","level":2,"score":0.4485826790332794},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.431709349155426},{"id":"https://openalex.org/C512399662","wikidata":"https://www.wikidata.org/wiki/Q3505712","display_name":"Family medicine","level":1,"score":0.37562617659568787},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.35349273681640625},{"id":"https://openalex.org/C19527891","wikidata":"https://www.wikidata.org/wiki/Q1120908","display_name":"Medical physics","level":1,"score":0.35332077741622925},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3469316363334656},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14912018179893494},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.13846713304519653},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.12952491641044617},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.12439507246017456},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.09719464182853699},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[{"descriptor_ui":"D010380","descriptor_name":"Peer Review","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D013223","descriptor_name":"Statistics as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D018156","descriptor_name":"Journalism, Medical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D032921","descriptor_name":"Consensus","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D035081","descriptor_name":"Editorial Policies","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":6,"locations":[{"id":"doi:10.1371/journal.pone.0061401","doi":"https://doi.org/10.1371/journal.pone.0061401","is_oa":true,"landing_page_url":"https://doi.org/10.1371/journal.pone.0061401","pdf_url":"https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0061401&type=printable","source":{"id":"https://openalex.org/S202381698","display_name":"PLoS ONE","issn_l":"1932-6203","issn":["1932-6203"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-c","doyens","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315706","host_organization_name":"Public Library of Science","host_organization_lineage":["https://openalex.org/P4310315706"],"host_organization_lineage_names":["Public Library of Science"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PLoS ONE","raw_type":"journal-article"},{"id":"pmh:oai:europepmc.org:2646162","doi":null,"is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/3642182","pdf_url":null,"source":{"id":"https://openalex.org/S4306400806","display_name":"Europe PMC (PubMed Central)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1303153112","host_organization_name":"European Bioinformatics Institute","host_organization_lineage":["https://openalex.org/I1303153112"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Text"},{"id":"pmh:oai:pubmedcentral.nih.gov:3642182","doi":null,"is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/3642182","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PLoS One","raw_type":"Text"},{"id":"pmid:23658692","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/23658692","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PloS one","raw_type":"Journal Article"},{"id":"pmh:oai:RePEc:plo:pone00:0061401","doi":null,"is_oa":false,"landing_page_url":"https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0061401","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"pmh:oai:doaj.org/article:bf7b9ba4520947048bdfb767e1ce3e3f","doi":null,"is_oa":false,"landing_page_url":"https://doaj.org/article/bf7b9ba4520947048bdfb767e1ce3e3f","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"PLoS ONE, Vol 8, Iss 5, p e61401 (2013)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1371/journal.pone.0061401","doi":"https://doi.org/10.1371/journal.pone.0061401","is_oa":true,"landing_page_url":"https://doi.org/10.1371/journal.pone.0061401","pdf_url":"https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0061401&type=printable","source":{"id":"https://openalex.org/S202381698","display_name":"PLoS ONE","issn_l":"1932-6203","issn":["1932-6203"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-c","doyens","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315706","host_organization_name":"Public Library of Science","host_organization_lineage":["https://openalex.org/P4310315706"],"host_organization_lineage_names":["Public Library of Science"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PLoS ONE","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W1972622142.pdf","grobid_xml":"https://content.openalex.org/works/W1972622142.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W1964602390","https://openalex.org/W1983897914","https://openalex.org/W1986759740","https://openalex.org/W1989926238","https://openalex.org/W1996290308","https://openalex.org/W2001662846","https://openalex.org/W2038579646","https://openalex.org/W2063743914","https://openalex.org/W2065873852","https://openalex.org/W2076061140","https://openalex.org/W2127384053","https://openalex.org/W2133386222","https://openalex.org/W2150290224","https://openalex.org/W2164777277","https://openalex.org/W2183036996","https://openalex.org/W2212838023","https://openalex.org/W2395817536","https://openalex.org/W2494206234"],"related_works":["https://openalex.org/W7098874957","https://openalex.org/W7095951780","https://openalex.org/W2065873852","https://openalex.org/W2035878678","https://openalex.org/W2416847083","https://openalex.org/W2127384053","https://openalex.org/W1969134897","https://openalex.org/W4250730631","https://openalex.org/W2212838023","https://openalex.org/W2591813246"],"abstract_inverted_index":{"BACKGROUND:":[0],"Peer":[1],"review":[2,22,57],"is":[3,14,207],"the":[4,31,34,55,194,200,215,221,245,253,284],"mainstay":[5],"of":[6,17,19,33,37,54,106,108,162,171,180,223,247,255],"editorial":[7,150,224],"decision":[8,151,225],"making":[9],"for":[10,126,132,178,220,273,276],"medical":[11,38,62],"journals.":[12,39],"There":[13],"a":[15,60,100,270],"dearth":[16],"evaluations":[18],"journal":[20],"peer":[21,56],"with":[23],"regard":[24],"to":[25,124,190,244,268],"reliability":[26],"and":[27,78,87,149,165,235],"validity,":[28],"particularly":[29],"in":[30,193,252],"light":[32],"wide":[35],"variety":[36],"Studies":[40],"carried":[41],"out":[42],"so":[43],"far":[44],"indicate":[45],"low":[46,174],"agreement":[47,102,121,143,206,212,237],"among":[48,103,217],"reviewers.":[49],"We":[50,227],"present":[51],"an":[52],"analysis":[53,254],"process":[58],"at":[59],"general":[61],"journal,":[63],"Deutsches":[64],"\u00c4rzteblatt":[65],"International.":[66],"METHODOLOGY/PRINCIPAL":[67],"FINDINGS:":[68],"554":[69],"reviewer":[70,147,156,163,205,262,265],"recommendations":[71,157,164,266],"on":[72,210],"206":[73],"manuscripts":[74,278],"submitted":[75],"between":[76,146],"7/2008":[77],"12/2009":[79],"were":[80,158],"analyzed:":[81],"7%":[82],"recommended":[83],"acceptance,":[84,127],"74%":[85],"revision":[86,285],"19%":[88],"rejection.":[89,133],"Concerning":[90],"acceptance":[91],"(with":[92],"or":[93,116,238,249,257],"without":[94],"revision)":[95],"versus":[96],"rejection,":[97],"there":[98],"was":[99,111,129,152,173],"substantial":[101,142],"reviewers":[104,218],"(74.3%":[105],"pairs":[107],"recommendations)":[109],"that":[110,204,229],"not":[112],"reflected":[113],"by":[114,169],"Fleiss'":[115,250],"Cohen's":[117,248],"kappa":[118,139],"(<0.2).":[119],"The":[120,160],"rate":[122],"amounted":[123],"84%":[125],"but":[128],"only":[130],"31%":[131],"An":[134],"alternative":[135,239],"kappa-statistic,":[136],"however,":[137],"Gwet's":[138],"(AC1),":[140],"indicated":[141],"(0.63).":[144],"Concordance":[145],"recommendation":[148],"almost":[153],"perfect":[154],"when":[155],"unanimous.":[159],"correlation":[161,176],"citations":[166,274],"as":[167,233],"counted":[168],"Web":[170],"Science":[172],"(partial":[175],"adjusted":[177],"year":[179],"publication:":[181],"-0.03,":[182],"n.s.).":[183],"CONCLUSIONS/SIGNIFICANCE:":[184],"Although":[185],"our":[186,196],"figures":[187],"are":[188,242],"similar":[189],"those":[191],"reported":[192],"literature":[195],"conclusion":[197],"differs":[198],"from":[199],"widely":[201],"held":[202],"view":[203],"low:":[208],"Based":[209],"overall":[211],"we":[213],"consider":[214],"concordance":[216],"sufficient":[219],"purposes":[222],"making.":[226],"believe":[228],"various":[230],"measures,":[231],"such":[232],"positive":[234],"negative":[236],"Kappa":[240,251],"values":[241],"superior":[243],"application":[246],"nominal":[256],"ordinal":[258],"level":[259],"data":[260],"regarding":[261],"agreement.":[263],"Also,":[264],"seem":[267],"be":[269,280],"poor":[271],"proxy":[272],"because,":[275],"example,":[277],"will":[279],"changed":[281],"considerably":[282],"during":[283],"process.":[286]},"counts_by_year":[{"year":2024,"cited_by_count":3},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":3},{"year":2015,"cited_by_count":7},{"year":2014,"cited_by_count":4},{"year":2013,"cited_by_count":2}],"updated_date":"2026-10-31T12:09:11.096373","created_date":"2016-06-24T00:00:00"}