{"id":"https://openalex.org/W2768281947","doi":"https://doi.org/10.1073/pnas.1707323114","title":"Reviewer bias in single- versus double-blind peer review","display_name":"Reviewer bias in single- versus double-blind peer review","publication_year":2017,"publication_date":"2017-11-14","ids":{"openalex":"https://openalex.org/W2768281947","doi":"https://doi.org/10.1073/pnas.1707323114","mag":"2768281947","pmid":"https://pubmed.ncbi.nlm.nih.gov/29138317"},"language":"en","primary_location":{"id":"doi:10.1073/pnas.1707323114","is_oa":true,"landing_page_url":"https://doi.org/10.1073/pnas.1707323114","pdf_url":"https://www.pnas.org/content/pnas/114/48/12708.full.pdf","source":{"id":"https://openalex.org/S125754415","display_name":"Proceedings of the National Academy of Sciences","issn_l":"0027-8424","issn":["0027-8424","1091-6490"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320052","host_organization_name":"National Academy of Sciences","host_organization_lineage":["https://openalex.org/P4310320052"],"host_organization_lineage_names":["National Academy of Sciences"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the National Academy of Sciences","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://www.pnas.org/content/pnas/114/48/12708.full.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068021191","display_name":"Andrew Tomkins","orcid":"https://orcid.org/0000-0002-1611-9255"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Andrew Tomkins","raw_affiliation_strings":["Google, Inc., Mountain View, CA 94043;"],"raw_orcid":"https://orcid.org/0000-0002-1611-9255","affiliations":[{"raw_affiliation_string":"Google, Inc., Mountain View, CA 94043;","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100402996","display_name":"Min Zhang","orcid":"https://orcid.org/0000-0003-3158-1920"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Zhang","raw_affiliation_strings":["State Key Laboratory of Intelligent Technology and Systems, Department of Computer Science and Technology, Tsinghua University, Beijing 100084, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Intelligent Technology and Systems, Department of Computer Science and Technology, Tsinghua University, Beijing 100084, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034924480","display_name":"William D. Heavlin","orcid":"https://orcid.org/0000-0003-0377-8988"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"William D. Heavlin","raw_affiliation_strings":["Google, Inc., Mountain View, CA 94043;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google, Inc., Mountain View, CA 94043;","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5068021191"],"corresponding_institution_ids":["https://openalex.org/I1291425158"],"apc_list":null,"apc_paid":null,"fwci":78.993,"has_fulltext":false,"cited_by_count":603,"citation_normalized_percentile":{"value":0.99921082,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"114","issue":"48","first_page":"12708","last_page":"12713"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12478","display_name":"Wikis in Education and Collaboration","score":0.9484999775886536,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/prestige","display_name":"Prestige","score":0.6887238025665283},{"id":"https://openalex.org/keywords/double-blind","display_name":"Double blind","score":0.6634290218353271},{"id":"https://openalex.org/keywords/single-blind","display_name":"Single blind","score":0.5720357894897461},{"id":"https://openalex.org/keywords/peer-review","display_name":"Peer review","score":0.5426758527755737},{"id":"https://openalex.org/keywords/cornerstone","display_name":"Cornerstone","score":0.5239684581756592},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.44183504581451416},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.4241287410259247},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3322391211986542},{"id":"https://openalex.org/keywords/alternative-medicine","display_name":"Alternative medicine","score":0.18000006675720215},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.17844372987747192},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.12471020221710205},{"id":"https://openalex.org/keywords/history","display_name":"History","score":0.07210636138916016},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.07198894023895264}],"concepts":[{"id":"https://openalex.org/C2778329345","wikidata":"https://www.wikidata.org/wiki/Q919978","display_name":"Prestige","level":2,"score":0.6887238025665283},{"id":"https://openalex.org/C2991744798","wikidata":"https://www.wikidata.org/wiki/Q608510","display_name":"Double blind","level":4,"score":0.6634290218353271},{"id":"https://openalex.org/C3018954396","wikidata":"https://www.wikidata.org/wiki/Q608510","display_name":"Single blind","level":3,"score":0.5720357894897461},{"id":"https://openalex.org/C138368954","wikidata":"https://www.wikidata.org/wiki/Q215028","display_name":"Peer review","level":2,"score":0.5426758527755737},{"id":"https://openalex.org/C2780616401","wikidata":"https://www.wikidata.org/wiki/Q1133673","display_name":"Cornerstone","level":2,"score":0.5239684581756592},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.44183504581451416},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4241287410259247},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3322391211986542},{"id":"https://openalex.org/C204787440","wikidata":"https://www.wikidata.org/wiki/Q188504","display_name":"Alternative medicine","level":2,"score":0.18000006675720215},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.17844372987747192},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.12471020221710205},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.07210636138916016},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.07198894023895264},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C27081682","wikidata":"https://www.wikidata.org/wiki/Q269829","display_name":"Placebo","level":3,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C141071460","wikidata":"https://www.wikidata.org/wiki/Q40821","display_name":"Surgery","level":1,"score":0.0},{"id":"https://openalex.org/C168563851","wikidata":"https://www.wikidata.org/wiki/Q1436668","display_name":"Randomized controlled trial","level":2,"score":0.0}],"mesh":[{"descriptor_ui":"D001319","descriptor_name":"Authorship","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D001319","descriptor_name":"Authorship","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D001319","descriptor_name":"Authorship","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D004311","descriptor_name":"Double-Blind Method","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D004311","descriptor_name":"Double-Blind Method","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D004311","descriptor_name":"Double-Blind Method","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015233","descriptor_name":"Models, Statistical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D015233","descriptor_name":"Models, Statistical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D015233","descriptor_name":"Models, Statistical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D016037","descriptor_name":"Single-Blind Method","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016037","descriptor_name":"Single-Blind Method","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016037","descriptor_name":"Single-Blind Method","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D017594","descriptor_name":"Publication Bias","qualifier_ui":"Q000639","qualifier_name":"trends","is_major_topic":false},{"descriptor_ui":"D017594","descriptor_name":"Publication Bias","qualifier_ui":"Q000639","qualifier_name":"trends","is_major_topic":false},{"descriptor_ui":"D017594","descriptor_name":"Publication Bias","qualifier_ui":"Q000639","qualifier_name":"trends","is_major_topic":false},{"descriptor_ui":"D017712","descriptor_name":"Peer Review, Research","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D017712","descriptor_name":"Peer Review, Research","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D017712","descriptor_name":"Peer Review, Research","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D017712","descriptor_name":"Peer Review, Research","qualifier_ui":"Q000941","qualifier_name":"ethics","is_major_topic":false},{"descriptor_ui":"D017712","descriptor_name":"Peer Review, Research","qualifier_ui":"Q000941","qualifier_name":"ethics","is_major_topic":false},{"descriptor_ui":"D017712","descriptor_name":"Peer Review, Research","qualifier_ui":"Q000941","qualifier_name":"ethics","is_major_topic":false},{"descriptor_ui":"D035081","descriptor_name":"Editorial Policies","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D035081","descriptor_name":"Editorial Policies","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D035081","descriptor_name":"Editorial Policies","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":3,"locations":[{"id":"doi:10.1073/pnas.1707323114","is_oa":true,"landing_page_url":"https://doi.org/10.1073/pnas.1707323114","pdf_url":"https://www.pnas.org/content/pnas/114/48/12708.full.pdf","source":{"id":"https://openalex.org/S125754415","display_name":"Proceedings of the National Academy of Sciences","issn_l":"0027-8424","issn":["0027-8424","1091-6490"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320052","host_organization_name":"National Academy of Sciences","host_organization_lineage":["https://openalex.org/P4310320052"],"host_organization_lineage_names":["National Academy of Sciences"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the National Academy of Sciences","raw_type":"journal-article"},{"id":"pmid:29138317","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/29138317","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the National Academy of Sciences of the United States of America","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:5715744","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/5715744","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"Proc Natl Acad Sci U S A","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1073/pnas.1707323114","is_oa":true,"landing_page_url":"https://doi.org/10.1073/pnas.1707323114","pdf_url":"https://www.pnas.org/content/pnas/114/48/12708.full.pdf","source":{"id":"https://openalex.org/S125754415","display_name":"Proceedings of the National Academy of Sciences","issn_l":"0027-8424","issn":["0027-8424","1091-6490"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320052","host_organization_name":"National Academy of Sciences","host_organization_lineage":["https://openalex.org/P4310320052"],"host_organization_lineage_names":["National Academy of Sciences"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the National Academy of Sciences","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3892115987","display_name":null,"funder_award_id":"61532011","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8096314617","display_name":"\u57fa\u4e8e\u4efb\u52a1\u7684\u8de8\u9886\u57df\u5f02\u8d28\u53ef\u89e3\u91ca\u7684\u63a8\u8350\u65b9\u6cd5\u7814\u7a76","funder_award_id":"61672311","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W1536128219","https://openalex.org/W1541009479","https://openalex.org/W1864634822","https://openalex.org/W1971814199","https://openalex.org/W1993156485","https://openalex.org/W2017899036","https://openalex.org/W2023754051","https://openalex.org/W2035745790","https://openalex.org/W2037305461","https://openalex.org/W2049657666","https://openalex.org/W2075551732","https://openalex.org/W2089603704","https://openalex.org/W2102073367","https://openalex.org/W2104410755","https://openalex.org/W2104629522","https://openalex.org/W2128127670","https://openalex.org/W2143628338","https://openalex.org/W2212838023","https://openalex.org/W2269267791","https://openalex.org/W2327379752","https://openalex.org/W2463347236","https://openalex.org/W2489734851","https://openalex.org/W2526355008","https://openalex.org/W2527516663","https://openalex.org/W2587486461","https://openalex.org/W4229893469","https://openalex.org/W4292887282"],"related_works":["https://openalex.org/W2321180484","https://openalex.org/W4387413063","https://openalex.org/W2994546353","https://openalex.org/W3097000576","https://openalex.org/W3029026403","https://openalex.org/W2495360025","https://openalex.org/W2525765130","https://openalex.org/W4249929441","https://openalex.org/W3136428523","https://openalex.org/W2321091178"],"abstract_inverted_index":{"Peer":[0],"review":[1,78,93],"may":[2],"be":[3],"\"single-blind,\"":[4],"in":[5,20,36,48,73,88,143],"which":[6,21,74,89],"reviewers":[7,100,170],"are":[8,101,117,165,171,196],"aware":[9],"of":[10,15,51,97,106,133],"the":[11,49,52,114,131,144],"names":[12],"and":[13,63,139,153,161,189,201],"affiliations":[14],"paper":[16],"authors,":[17,186],"or":[18,34],"\"double-blind,\"":[19],"this":[22],"information":[23,127],"is":[24],"hidden.":[25],"Noting":[26],"that":[27],"computer":[28],"science":[29],"research":[30],"often":[31],"appears":[32],"first":[33],"exclusively":[35],"peer-reviewed":[37],"conferences":[38],"rather":[39],"than":[40,175],"journals,":[41],"we":[42],"study":[43],"these":[44,98],"two":[45,116],"reviewing":[46,137],"models":[47],"context":[50],"10th":[53],"Association":[54],"for":[55,81,135,149,156,181],"Computing":[56],"Machinery":[57],"International":[58],"Conference":[59],"on":[60],"Web":[61],"Search":[62],"Data":[64],"Mining,":[65],"a":[66,85,104,120],"highly":[67],"selective":[68],"venue":[69],"(15.6%":[70],"acceptance":[71,182],"rate)":[72],"expert":[75],"committee":[76,91,107],"members":[77,92,108],"full-length":[79],"submissions":[80],"acceptance.":[82],"We":[83],"present":[84],"controlled":[86],"experiment":[87],"four":[90,99],"each":[94],"paper.":[95],"Two":[96],"drawn":[102,118],"from":[103,119,158,184],"pool":[105,122],"with":[109],"access":[110],"to":[111,167,179],"author":[112],"information;":[113],"other":[115],"disjoint":[121],"without":[123],"such":[124],"access.":[125],"This":[126],"asymmetry":[128],"persists":[129],"through":[130],"process":[132],"bidding":[134],"papers,":[136,138],"entering":[140],"scores.":[141],"Reviewers":[142],"single-blind":[145,169],"condition":[146],"typically":[147],"bid":[148,155],"22%":[150],"fewer":[151],"papers":[152,157,164,183],"preferentially":[154],"top":[159,187,190],"universities":[160],"companies.":[162,191],"Once":[163],"allocated":[166],"reviewers,":[168],"significantly":[172],"more":[173],"likely":[174],"their":[176],"double-blind":[177],"counterparts":[178],"recommend":[180],"famous":[185],"universities,":[188],"The":[192],"estimated":[193],"odds":[194],"multipliers":[195],"tangible,":[197],"at":[198],"1.63,":[199],"1.58,":[200],"2.10,":[202],"respectively.":[203]},"counts_by_year":[{"year":2026,"cited_by_count":26},{"year":2025,"cited_by_count":57},{"year":2024,"cited_by_count":70},{"year":2023,"cited_by_count":91},{"year":2022,"cited_by_count":109},{"year":2021,"cited_by_count":94},{"year":2020,"cited_by_count":77},{"year":2019,"cited_by_count":47},{"year":2018,"cited_by_count":30},{"year":2017,"cited_by_count":2}],"updated_date":"2026-06-06T09:05:17.133730","created_date":"2025-10-10T00:00:00"}