{"id":"https://openalex.org/W7223059352","doi":"https://doi.org/10.31234/osf.io/t968e_v4","title":"What Pilot Studies Can (and Cannot) Do for Validity in Psychological Research","display_name":"What Pilot Studies Can (and Cannot) Do for Validity in Psychological Research","publication_year":2026,"publication_date":"2026-10-20","ids":{"openalex":"https://openalex.org/W7223059352","doi":"https://doi.org/10.31234/osf.io/t968e_v4"},"language":"en","primary_location":{"id":"doi:10.31234/osf.io/t968e_v4","doi":"https://doi.org/10.31234/osf.io/t968e_v4","is_oa":true,"landing_page_url":"https://doi.org/10.31234/osf.io/t968e_v4","pdf_url":null,"source":{"id":"https://openalex.org/S4306401687","display_name":"PsyArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.31234/osf.io/t968e_v4","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088062663","display_name":"Yashvin Seetahul","orcid":"https://orcid.org/0000-0001-7487-3398","observed_orcids":["https://orcid.org/0000-0001-7487-3398"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yashvin Seetahul","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-7487-3398","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087195372","display_name":"Mahmoud Medhat Elsherif","orcid":"https://orcid.org/0000-0002-0540-3998","observed_orcids":["https://orcid.org/0000-0002-0540-3998"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mahmoud Medhat Elsherif","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-0540-3998","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056348040","display_name":"Caroline Zygar\u2013Hoffmann","orcid":"https://orcid.org/0000-0002-8677-2276","observed_orcids":["https://orcid.org/0000-0002-8677-2276"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Caroline Zygar-Hoffmann","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074635467","display_name":"Lukas Wallrich","orcid":"https://orcid.org/0000-0003-2121-5177","observed_orcids":["https://orcid.org/0000-0003-2121-5177"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lukas Wallrich","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-2121-5177","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016073136","display_name":"Priya Silverstein","orcid":"https://orcid.org/0000-0003-0095-339X","observed_orcids":["https://orcid.org/0000-0003-0095-339X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Priya Silverstein","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083516845","display_name":"Bj\u00f8rn S\u00e6trevik","orcid":"https://orcid.org/0000-0002-9367-6987","observed_orcids":["https://orcid.org/0000-0002-9367-6987"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bj\u00f8rn S\u00e6trevik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069790954","display_name":"Ilse L. Pit","orcid":"https://orcid.org/0000-0002-4066-8086","observed_orcids":["https://orcid.org/0000-0002-4066-8086"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ilse L. Pit","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-4066-8086","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046474955","display_name":"Hannah D. Loenneker","orcid":"https://orcid.org/0000-0003-0327-6507","observed_orcids":["https://orcid.org/0000-0003-0327-6507"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hannah Loenneker","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-0327-6507","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049483283","display_name":"Neele Henriette Heiser","orcid":"https://orcid.org/0000-0003-4654-4129","observed_orcids":["https://orcid.org/0000-0003-4654-4129"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Neele Henriette Heiser","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-4654-4129","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035784592","display_name":"Isaac J Handley-Miner","orcid":"https://orcid.org/0000-0002-9941-2624","observed_orcids":["https://orcid.org/0000-0002-9941-2624"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Isaac J Handley-Miner","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091046580","display_name":"Christopher James Graham","orcid":"https://orcid.org/0000-0002-1144-7970","observed_orcids":["https://orcid.org/0000-0002-1144-7970"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Christopher James Graham","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-1144-7970","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121735768","display_name":"Yu Yang Chou","orcid":"https://orcid.org/0009-0002-3988-0917","observed_orcids":["https://orcid.org/0009-0002-3988-0917"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu Yang Chou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008207018","display_name":"Brett Buttliere","orcid":"https://orcid.org/0000-0001-5025-0460","observed_orcids":["https://orcid.org/0000-0001-5025-0460"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Brett Buttliere","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022707855","display_name":"Agata Bochy\u0144ska","orcid":"https://orcid.org/0000-0001-6211-8600","observed_orcids":["https://orcid.org/0000-0001-6211-8600"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Agata Bochynska","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-6211-8600","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033307361","display_name":"Julia Beitner","orcid":"https://orcid.org/0000-0002-2539-7011","observed_orcids":["https://orcid.org/0000-0002-2539-7011","https://orcid.org/0000-0003-4029-9847"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Julia Beitner","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5051896322","display_name":"Mary Beth Neff","orcid":"https://orcid.org/0000-0002-9549-5936","observed_orcids":["https://orcid.org/0000-0002-9549-5936"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mary Beth Neff","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-9549-5936","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13284","display_name":"Psychology Research and Bibliometrics","score":0.38530001044273376,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13284","display_name":"Psychology Research and Bibliometrics","score":0.38530001044273376,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.322299987077713,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.05649999901652336,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pilot-study","display_name":"pilot study","score":0.9909999966621399},{"id":"https://openalex.org/keywords/research-validity","display_name":"research validity","score":0.7519999742507935},{"id":"https://openalex.org/keywords/construct-validity","display_name":"construct validity","score":0.8640000224113464},{"id":"https://openalex.org/keywords/internal-validity","display_name":"internal validity","score":0.9980000257492065},{"id":"https://openalex.org/keywords/generalizability","display_name":"generalizability","score":1.0},{"id":"https://openalex.org/keywords/statistical-conclusion-validity","display_name":"statistical conclusion validity","score":0.8790000081062317},{"id":"https://openalex.org/keywords/study-design","display_name":"research design","score":0.3889999985694885},{"id":"https://openalex.org/keywords/measurement-validation","display_name":"measurement validation","score":0.593999981880188},{"id":"https://openalex.org/keywords/reporting-transparency","display_name":"reporting transparency","score":0.5600000023841858}],"concepts":[{"id":"https://openalex.org/C2780224610","wikidata":"https://www.wikidata.org/wiki/Q1530061","display_name":"Credibility","level":2,"score":0.6507999897003174},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.63919997215271},{"id":"https://openalex.org/C51082289","wikidata":"https://www.wikidata.org/wiki/Q4113800","display_name":"Internal validity","level":2,"score":0.5745999813079834},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.5651000142097473},{"id":"https://openalex.org/C174106493","wikidata":"https://www.wikidata.org/wiki/Q1057880","display_name":"External validity","level":2,"score":0.5573999881744385},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5357000231742859},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.5},{"id":"https://openalex.org/C2777363581","wikidata":"https://www.wikidata.org/wiki/Q15098235","display_name":"Harm","level":2,"score":0.49559998512268066},{"id":"https://openalex.org/C2779318504","wikidata":"https://www.wikidata.org/wiki/Q1438035","display_name":"Research design","level":2,"score":0.475600004196167},{"id":"https://openalex.org/C188255311","wikidata":"https://www.wikidata.org/wiki/Q7256388","display_name":"Psychological research","level":2,"score":0.44429999589920044},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.38519999384880066},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.38350000977516174},{"id":"https://openalex.org/C182050348","wikidata":"https://www.wikidata.org/wiki/Q5186576","display_name":"Criterion validity","level":4,"score":0.3391000032424927},{"id":"https://openalex.org/C90963130","wikidata":"https://www.wikidata.org/wiki/Q6015484","display_name":"Incremental validity","level":4,"score":0.32749998569488525},{"id":"https://openalex.org/C70364389","wikidata":"https://www.wikidata.org/wiki/Q18757","display_name":"Validity","level":3,"score":0.326200008392334},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.32600000500679016},{"id":"https://openalex.org/C20685875","wikidata":"https://www.wikidata.org/wiki/Q7239678","display_name":"Predictive validity","level":2,"score":0.321399986743927},{"id":"https://openalex.org/C2779820079","wikidata":"https://www.wikidata.org/wiki/Q1427962","display_name":"Diligence","level":2,"score":0.30809998512268066},{"id":"https://openalex.org/C71008984","wikidata":"https://www.wikidata.org/wiki/Q2890076","display_name":"Rigour","level":2,"score":0.3041999936103821},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.30140000581741333},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.2953000068664551},{"id":"https://openalex.org/C2781016034","wikidata":"https://www.wikidata.org/wiki/Q794134","display_name":"Due diligence","level":2,"score":0.2847000062465668},{"id":"https://openalex.org/C108154423","wikidata":"https://www.wikidata.org/wiki/Q1469792","display_name":"Salience (neuroscience)","level":2,"score":0.28139999508857727},{"id":"https://openalex.org/C163515075","wikidata":"https://www.wikidata.org/wiki/Q873512","display_name":"Psychological testing","level":2,"score":0.2581999897956848},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.25290000438690186},{"id":"https://openalex.org/C116211729","wikidata":"https://www.wikidata.org/wiki/Q5372350","display_name":"Test validity","level":3,"score":0.2515999972820282}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.31234/osf.io/t968e_v4","doi":"https://doi.org/10.31234/osf.io/t968e_v4","is_oa":true,"landing_page_url":"https://doi.org/10.31234/osf.io/t968e_v4","pdf_url":null,"source":{"id":"https://openalex.org/S4306401687","display_name":"PsyArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.31234/osf.io/t968e_v4","doi":"https://doi.org/10.31234/osf.io/t968e_v4","is_oa":true,"landing_page_url":"https://doi.org/10.31234/osf.io/t968e_v4","pdf_url":null,"source":{"id":"https://openalex.org/S4306401687","display_name":"PsyArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7134960110","https://openalex.org/W4413154917","https://openalex.org/W7223559979","https://openalex.org/W7046454628","https://openalex.org/W4383599115","https://openalex.org/W2580311554","https://openalex.org/W2142589129","https://openalex.org/W7107949366","https://openalex.org/W2602239210","https://openalex.org/W1999204192"],"abstract_inverted_index":{"Recent":[0],"debates":[1],"about":[2,26],"psychology\u2019s":[3],"credibility":[4],"have":[5],"renewed":[6],"attention":[7],"to":[8,34,119,131,156],"validity":[9,70,90,173],"threats.":[10],"We":[11,38,123],"argue":[12,125],"that":[13,86,126],"pilot":[14,108,159],"studies":[15],"can":[16,57,88],"function":[17,65],"as":[18,66,93,142],"methodological":[19],"due":[20],"diligence":[21],"by":[22],"improving":[23],"early":[24],"decisions":[25],"design,":[27],"measurement,":[28],"manipulation,":[29],"sampling,":[30],"and":[31,47,63,72,79,148],"analysis":[32],"prior":[33],"a":[35,94,166],"main":[36],"study.":[37],"examine":[39],"four":[40],"types":[41],"of":[42,144,165,174],"validity:":[43],"construct,":[44],"internal,":[45],"external,":[46],"statistical-conclusion":[48],"validity.":[49],"Across":[50],"these":[51],"domains,":[52],"we":[53,83],"show":[54],"how":[55,158],"piloting":[56,87],"help":[58],"researchers":[59],"assess":[60,171],"whether":[61],"materials":[62],"procedures":[64],"intended,":[67],"identify":[68],"potential":[69],"threats,":[71],"provide":[73],"key":[74],"design":[75,99,134,164],"parameters":[76],"for":[77],"planning":[78],"model":[80],"choice.":[81],"However,":[82],"also":[84],"caution":[85],"harm":[89],"when":[91,113,127],"treated":[92,141],"consequence-free":[95],"space":[96],"in":[97],"which":[98],"or":[100,112,135],"analytic":[101,136],"choices":[102],"are":[103,117,129],"made":[104],"without":[105],"disclosing":[106],"the":[107,145,162,172],"evidence":[109,160],"behind":[110],"them,":[111],"small,":[114],"noisy":[115],"pilots":[116,128],"used":[118,130],"justify":[120,132],"overconfident":[121],"inferences.":[122],"therefore":[124],"substantive":[133],"choices,":[137],"they":[138],"should":[139],"be":[140],"part":[143],"evidential":[146],"record":[147],"reported":[149],"transparently.":[150],"Greater":[151],"transparency":[152],"would":[153],"enable":[154],"readers":[155],"evaluate":[157],"shaped":[161],"final":[163],"study":[167],"and,":[168],"consequently,":[169],"better":[170],"its":[175],"findings.":[176]},"counts_by_year":[],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2026-10-21T00:00:00"}