{"id":"https://openalex.org/W4416163441","doi":"https://doi.org/10.31235/osf.io/4gqce_v3","title":"Reliability, bias and randomisation in peer review: a simulation","display_name":"Reliability, bias and randomisation in peer review: a simulation","publication_year":2025,"publication_date":"2025-11-13","ids":{"openalex":"https://openalex.org/W4416163441","doi":"https://doi.org/10.31235/osf.io/4gqce_v3"},"language":"en","primary_location":{"id":"doi:10.31235/osf.io/4gqce_v3","doi":"https://doi.org/10.31235/osf.io/4gqce_v3","is_oa":true,"landing_page_url":"https://doi.org/10.31235/osf.io/4gqce_v3","pdf_url":null,"source":{"id":"https://openalex.org/S4306401238","display_name":"SocArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.31235/osf.io/4gqce_v3","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119464459","display_name":"Alexander Hulkes","orcid":"https://orcid.org/0009-0005-0521-8572","observed_orcids":["https://orcid.org/0009-0005-0521-8572"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alexander Hulkes","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0009-0005-0521-8572","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120348990","display_name":"Cillian Brophy","orcid":"https://orcid.org/0009-0008-2465-0880","observed_orcids":["https://orcid.org/0009-0008-2465-0880"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cillian Brophy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5105473329","display_name":"Benjamin Steyn","orcid":"https://orcid.org/0009-0005-2009-0829","observed_orcids":["https://orcid.org/0009-0005-2009-0829"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ben Steyn","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14185","display_name":"Research, Science, and Academia","score":0.39579999446868896,"subfield":{"id":"https://openalex.org/subfields/1405","display_name":"Management of Technology and Innovation"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T14185","display_name":"Research, Science, and Academia","score":0.39579999446868896,"subfield":{"id":"https://openalex.org/subfields/1405","display_name":"Management of Technology and Innovation"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10102","display_name":"scientometrics and bibliometrics research","score":0.271699994802475,"subfield":{"id":"https://openalex.org/subfields/3309","display_name":"Library and Information Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13284","display_name":"Psychology Research and Bibliometrics","score":0.06610000133514404,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/peer-review","display_name":"peer review","score":0.8669999837875366},{"id":"https://openalex.org/keywords/research-funding-decisions","display_name":"research funding decisions","score":0.8370000123977661},{"id":"https://openalex.org/keywords/decision-reliability","display_name":"decision reliability","score":0.8579999804496765},{"id":"https://openalex.org/keywords/bias","display_name":"bias","score":0.5400000214576721},{"id":"https://openalex.org/keywords/research-proposal-assessment","display_name":"research proposal assessment","score":0.6589999794960022},{"id":"https://openalex.org/keywords/f1-score","display_name":"F1 score","score":0.753000020980835}],"concepts":[{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6931999921798706},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5127999782562256},{"id":"https://openalex.org/C27415008","wikidata":"https://www.wikidata.org/wiki/Q7256382","display_name":"Psychological intervention","level":2,"score":0.46880000829696655},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.46619999408721924},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4343000054359436},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.4009999930858612},{"id":"https://openalex.org/C2780033181","wikidata":"https://www.wikidata.org/wiki/Q24962805","display_name":"Decision aids","level":3,"score":0.38989999890327454},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.3684999942779541},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3465000092983246},{"id":"https://openalex.org/C186116695","wikidata":"https://www.wikidata.org/wiki/Q5249226","display_name":"Decision analysis","level":2,"score":0.3330000042915344},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.310699999332428},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.30720001459121704},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.29829999804496765},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.28519999980926514},{"id":"https://openalex.org/C95190672","wikidata":"https://www.wikidata.org/wiki/Q815382","display_name":"Meta-analysis","level":2,"score":0.2775999903678894},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.27709999680519104},{"id":"https://openalex.org/C112758219","wikidata":"https://www.wikidata.org/wiki/Q16038819","display_name":"Duration (music)","level":2,"score":0.2680000066757202},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.2667999863624573},{"id":"https://openalex.org/C59594135","wikidata":"https://www.wikidata.org/wiki/Q5249242","display_name":"Decision model","level":2,"score":0.26660001277923584},{"id":"https://openalex.org/C107327155","wikidata":"https://www.wikidata.org/wiki/Q330268","display_name":"Decision support system","level":2,"score":0.26589998602867126},{"id":"https://openalex.org/C84839998","wikidata":"https://www.wikidata.org/wiki/Q5249245","display_name":"Decision rule","level":2,"score":0.2655999958515167},{"id":"https://openalex.org/C2779318504","wikidata":"https://www.wikidata.org/wiki/Q1438035","display_name":"Research design","level":2,"score":0.2619999945163727},{"id":"https://openalex.org/C28901747","wikidata":"https://www.wikidata.org/wiki/Q177571","display_name":"Decision theory","level":2,"score":0.26080000400543213},{"id":"https://openalex.org/C84525736","wikidata":"https://www.wikidata.org/wiki/Q831366","display_name":"Decision tree","level":2,"score":0.2531000077724457},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.25040000677108765}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.31235/osf.io/4gqce_v3","doi":"https://doi.org/10.31235/osf.io/4gqce_v3","is_oa":true,"landing_page_url":"https://doi.org/10.31235/osf.io/4gqce_v3","pdf_url":null,"source":{"id":"https://openalex.org/S4306401238","display_name":"SocArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},{"id":"pmh:oai:share.osf.io:207808b3-cf7e-4dd3-996e-45e1c02d3128","doi":null,"is_oa":true,"landing_page_url":"https://osf.io/4gqce","pdf_url":null,"source":{"id":"https://openalex.org/S4306401238","display_name":"SocArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.31235/osf.io/4gqce_v3","doi":"https://doi.org/10.31235/osf.io/4gqce_v3","is_oa":true,"landing_page_url":"https://doi.org/10.31235/osf.io/4gqce_v3","pdf_url":null,"source":{"id":"https://openalex.org/S4306401238","display_name":"SocArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7220832649","https://openalex.org/W4407437168","https://openalex.org/W7006731699","https://openalex.org/W3162364639","https://openalex.org/W6977918013","https://openalex.org/W3187623100","https://openalex.org/W6929654300","https://openalex.org/W6950555609","https://openalex.org/W2325769886","https://openalex.org/W4401877909"],"abstract_inverted_index":{"For":[0],"a":[1,6,12,51,66,70,131],"variety":[2],"of":[3,21,43,60,69,90,130,170,173,185,192,194],"reasons,":[4],"including":[5],"need":[7],"to":[8,14,25,80,112,149,153],"save":[9],"time":[10],"and":[11,93,105,128,147,191],"desire":[13],"reduce":[15,113,119],"biases":[16],"in":[17,29,53,84,144,161],"outcomes,":[18],"some":[19],"funders":[20],"research":[22,71,188],"have":[23,39],"started":[24],"use":[26],"partial":[27,154],"randomisation":[28,37,92,108,155],"their":[30,54,61],"funding":[31,189],"decision":[32,86,91,120,136],"processes.":[33],"The":[34,164],"effect":[35],"that":[36],"interventions":[38],"on":[40,102],"the":[41,82,110,125,171,183],"reliability":[42,121],"those":[44],"processes":[45],"should,":[46],"it":[47,116,157],"is":[48,141,158],"argued,":[49],"be":[50,99,150],"consideration":[52],"use,":[55],"but":[56,115],"this":[57],"key":[58],"aspect":[59],"implementation":[62],"remains":[63],"under-appreciated.":[64],"Using":[65],"simple":[67,165],"specification":[68,169],"proposal":[72],"peer":[73],"review":[74],"process,":[75],"simulations":[76],"are":[77],"carried":[78],"out":[79],"explore":[81],"ways":[83],"which":[85],"reliability,":[87],"bias,":[88,114],"extent":[89],"other":[94],"factors":[95],"interact.":[96],"As":[97],"might":[98],"expected,":[100],"based":[101],"both":[103],"logic":[104],"existing":[106],"knowledge,":[107],"has":[109],"potential":[111],"may":[117,178],"also":[118,142,179],"as":[122,156],"inferred":[123],"from":[124],"F1":[126],"score":[127],"accuracy":[129],"simulated":[132],"binary":[133],"(successful,":[134],"rejected)":[135],"outcome":[137],"classification":[138],"process.":[139],"Bias":[140],"found,":[143],"one":[145],"sense":[146],"qualitatively,":[148],"rather":[151],"insensitive":[152],"typically":[159],"applied":[160],"real-world":[162],"situations.":[163],"yet":[166],"apparently":[167],"effective":[168],"simulation":[172],"reviewer":[174],"scores":[175],"implemented":[176],"here":[177],"provide":[180],"insights":[181],"into":[182],"distribution":[184],"merit":[186],"across":[187],"proposals,":[190],"assessment":[193],"them.":[195]},"counts_by_year":[],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2025-10-10T00:00:00"}