{"id":"https://openalex.org/W7225399532","doi":"https://doi.org/10.1186/s12874-026-02987-x","title":"The influence of sample size, modelling algorithms, and hyperparameters on bootstrap-based prediction stability in clinical prediction model development: An empirical study using the HOPE dataset","display_name":"The influence of sample size, modelling algorithms, and hyperparameters on bootstrap-based prediction stability in clinical prediction model development: An empirical study using the HOPE dataset","publication_year":2026,"publication_date":"2026-11-07","ids":{"openalex":"https://openalex.org/W7225399532","doi":"https://doi.org/10.1186/s12874-026-02987-x"},"language":"en","primary_location":{"id":"doi:10.1186/s12874-026-02987-x","doi":"https://doi.org/10.1186/s12874-026-02987-x","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s12874-026-02987-x","pdf_url":null,"source":{"id":"https://openalex.org/S185874209","display_name":"BMC Medical Research Methodology","issn_l":"1471-2288","issn":["1471-2288"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-b","doyens","jufo-1","ki-jl-1","medline","norway-1","oarl","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Medical Research Methodology","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1186/s12874-026-02987-x","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034187126","display_name":"Phichayut Phinyo","orcid":"https://orcid.org/0000-0002-8543-6254","observed_orcids":["https://orcid.org/0000-0002-8543-6254"]},"institutions":[{"id":"https://openalex.org/I183935753","display_name":"King's College London","ror":"https://ror.org/0220mzb33","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I183935753"]},{"id":"https://openalex.org/I48076826","display_name":"Chiang Mai University","ror":"https://ror.org/05m2fqn25","country_code":"TH","type":"education","lineage":["https://openalex.org/I48076826"]}],"countries":["GB","TH"],"is_corresponding":false,"raw_author_name":"Phichayut Phinyo","raw_affiliation_strings":["Centre for Clinical Epidemiology and Clinical Statistics, Faculty of Medicine, Chiang Mai University, Chiang Mai, Thailand","Department of Biomedical Informatics and Clinical Epidemiology (BioCE), Faculty of Medicine, Chiang Mai University, Chiang Mai, Thailand","Department of Biostatistics & Health Informatics, Institute of Psychiatry, Psychology & Neuroscience, King\u2019s College London, London, UK"],"raw_orcid":"https://orcid.org/0000-0002-8543-6254","affiliations":[{"raw_affiliation_string":"Centre for Clinical Epidemiology and Clinical Statistics, Faculty of Medicine, Chiang Mai University, Chiang Mai, Thailand","institution_ids":["https://openalex.org/I48076826"]},{"raw_affiliation_string":"Department of Biomedical Informatics and Clinical Epidemiology (BioCE), Faculty of Medicine, Chiang Mai University, Chiang Mai, Thailand","institution_ids":["https://openalex.org/I48076826"]},{"raw_affiliation_string":"Department of Biostatistics & Health Informatics, Institute of Psychiatry, Psychology & Neuroscience, King\u2019s College London, London, UK","institution_ids":["https://openalex.org/I183935753"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082274535","display_name":"Daniel R. Stahl","orcid":"https://orcid.org/0000-0001-7987-6619","observed_orcids":["https://orcid.org/0000-0001-7987-6619"]},"institutions":[{"id":"https://openalex.org/I183935753","display_name":"King's College London","ror":"https://ror.org/0220mzb33","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I183935753"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Daniel Stahl","raw_affiliation_strings":["Department of Biostatistics & Health Informatics, Institute of Psychiatry, Psychology & Neuroscience, King\u2019s College London, London, UK"],"raw_orcid":"https://orcid.org/0000-0001-7987-6619","affiliations":[{"raw_affiliation_string":"Department of Biostatistics & Health Informatics, Institute of Psychiatry, Psychology & Neuroscience, King\u2019s College London, London, UK","institution_ids":["https://openalex.org/I183935753"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020392846","display_name":"Ewan Carr","orcid":"https://orcid.org/0000-0002-1146-4922","observed_orcids":["https://orcid.org/0000-0002-1146-4922"]},"institutions":[{"id":"https://openalex.org/I183935753","display_name":"King's College London","ror":"https://ror.org/0220mzb33","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I183935753"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Ewan Carr","raw_affiliation_strings":["Department of Biostatistics & Health Informatics, Institute of Psychiatry, Psychology & Neuroscience, King\u2019s College London, London, UK"],"raw_orcid":"https://orcid.org/0000-0002-1146-4922","affiliations":[{"raw_affiliation_string":"Department of Biostatistics & Health Informatics, Institute of Psychiatry, Psychology & Neuroscience, King\u2019s College London, London, UK","institution_ids":["https://openalex.org/I183935753"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5020392846"],"corresponding_institution_ids":["https://openalex.org/I183935753"],"apc_list":{"value":3090,"currency":"USD","value_usd":3090},"apc_paid":{"value":3090,"currency":"USD","value_usd":3090},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14374","display_name":"Statistical Methods in Epidemiology","score":0.8794000148773193,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14374","display_name":"Statistical Methods in Epidemiology","score":0.8794000148773193,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11396","display_name":"Artificial Intelligence in Healthcare","score":0.06729999929666519,"subfield":{"id":"https://openalex.org/subfields/3605","display_name":"Health Information Management"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.014700000174343586,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/prediction-stability","display_name":"prediction stability","score":0.9470000267028809},{"id":"https://openalex.org/keywords/clinical-prediction-model","display_name":"clinical prediction model","score":0.9649999737739563},{"id":"https://openalex.org/keywords/sample-size","display_name":"sample size","score":0.9660000205039978},{"id":"https://openalex.org/keywords/hyperparameter-optimization","display_name":"hyperparameter optimization","score":0.9710000157356262},{"id":"https://openalex.org/keywords/lasso-logistic-regression","display_name":"LASSO logistic regression","score":0.9169999957084656},{"id":"https://openalex.org/keywords/random-forest","display_name":"random forest","score":0.9990000128746033},{"id":"https://openalex.org/keywords/xgboost","display_name":"XGBoost","score":1.0},{"id":"https://openalex.org/keywords/model-calibration","display_name":"model calibration","score":0.7129999995231628},{"id":"https://openalex.org/keywords/discrimination-statistics","display_name":"discrimination (statistics)","score":0.8650000095367432}],"concepts":[{"id":"https://openalex.org/C37616216","wikidata":"https://www.wikidata.org/wiki/Q3218363","display_name":"Lasso (programming language)","level":2,"score":0.6184999942779541},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.6171000003814697},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.5892000198364258},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.5814999938011169},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.5674999952316284},{"id":"https://openalex.org/C169258074","wikidata":"https://www.wikidata.org/wiki/Q245748","display_name":"Random forest","level":2,"score":0.5425999760627747},{"id":"https://openalex.org/C151956035","wikidata":"https://www.wikidata.org/wiki/Q1132755","display_name":"Logistic regression","level":2,"score":0.5167999863624573},{"id":"https://openalex.org/C45804977","wikidata":"https://www.wikidata.org/wiki/Q7239673","display_name":"Predictive modelling","level":2,"score":0.5103999972343445},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.5},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.44519999623298645},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.43290001153945923},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4260999858379364},{"id":"https://openalex.org/C83546350","wikidata":"https://www.wikidata.org/wiki/Q1139051","display_name":"Regression","level":2,"score":0.3885999917984009},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.33379998803138733},{"id":"https://openalex.org/C103402496","wikidata":"https://www.wikidata.org/wiki/Q1106171","display_name":"Prediction interval","level":2,"score":0.31610000133514404},{"id":"https://openalex.org/C139945424","wikidata":"https://www.wikidata.org/wiki/Q1940696","display_name":"Mean squared error","level":2,"score":0.3100999891757965},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.30559998750686646},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.30550000071525574},{"id":"https://openalex.org/C48921125","wikidata":"https://www.wikidata.org/wiki/Q10861030","display_name":"Linear regression","level":2,"score":0.29899999499320984},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.29019999504089355},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.28949999809265137},{"id":"https://openalex.org/C117251300","wikidata":"https://www.wikidata.org/wiki/Q1849855","display_name":"Parametric statistics","level":2,"score":0.2858000099658966},{"id":"https://openalex.org/C163175372","wikidata":"https://www.wikidata.org/wiki/Q3339222","display_name":"Linear model","level":2,"score":0.2777000069618225},{"id":"https://openalex.org/C168743327","wikidata":"https://www.wikidata.org/wiki/Q1826427","display_name":"Random effects model","level":3,"score":0.2669999897480011},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.2644999921321869},{"id":"https://openalex.org/C84525736","wikidata":"https://www.wikidata.org/wiki/Q831366","display_name":"Decision tree","level":2,"score":0.2603999972343445},{"id":"https://openalex.org/C24574437","wikidata":"https://www.wikidata.org/wiki/Q7135228","display_name":"Parametric model","level":3,"score":0.2565999925136566}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1186/s12874-026-02987-x","doi":"https://doi.org/10.1186/s12874-026-02987-x","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s12874-026-02987-x","pdf_url":null,"source":{"id":"https://openalex.org/S185874209","display_name":"BMC Medical Research Methodology","issn_l":"1471-2288","issn":["1471-2288"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-b","doyens","jufo-1","ki-jl-1","medline","norway-1","oarl","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Medical Research Methodology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1186/s12874-026-02987-x","doi":"https://doi.org/10.1186/s12874-026-02987-x","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s12874-026-02987-x","pdf_url":null,"source":{"id":"https://openalex.org/S185874209","display_name":"BMC Medical Research Methodology","issn_l":"1471-2288","issn":["1471-2288"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-b","doyens","jufo-1","ki-jl-1","medline","norway-1","oarl","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Medical Research Methodology","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Good health and well-being","id":"https://metadata.un.org/sdg/3","score":0.8371999859809875}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7163866825","https://openalex.org/W4384663192","https://openalex.org/W7224025940","https://openalex.org/W4389904066","https://openalex.org/W7166532361","https://openalex.org/W7167576429","https://openalex.org/W4409783558","https://openalex.org/W7213520090","https://openalex.org/W7128784320","https://openalex.org/W4416887894"],"abstract_inverted_index":{"Abstract":[0],"Background":[1],"The":[2],"stability":[3,73,166,301,308,330],"of":[4,44,59,74,117,124,132,202,258],"clinical":[5,28,36,187,303,353],"prediction":[6,48,101,177,300,304,354],"models":[7],"has":[8],"emerged":[9],"as":[10,276],"an":[11],"important":[12],"factor":[13],"in":[14,35,47,247,302,334],"model":[15,102,305,355],"development.":[16,306,356],"Models":[17,153],"producing":[18],"unstable":[19],"or":[20],"unreliable":[21],"predictions":[22,203,259],"can":[23],"ultimately":[24],"lead":[25],"to":[26,98,232,310,350],"improper":[27],"decisions":[29,62],"and":[30,83,87,151,158,163,180,198,228,240,263,279,322,346],"harm":[31],"patients":[32],"when":[33],"applied":[34],"practice.":[37],"Previous":[38],"studies":[39],"have":[40],"highlighted":[41],"the":[42,57,72,93,115,174,215],"role":[43,58],"sample":[45,76,118,190,236,270,315],"size":[46,217,316],"stability,":[49],"with":[50,142,194,204,210,260,285],"larger":[51,189],"samples":[52],"delivering":[53],"increased":[54],"stability.":[55],"However,":[56],"other":[60],"modelling":[61,78,134,144,320],"remains":[63],"unclear.":[64],"This":[65],"study":[66],"considered":[67],"three":[68,143],"modifiable":[69],"factors":[70],"influencing":[71],"predictions:":[75],"size,":[77,119],"algorithm":[79],"(i.e.,":[80],"parametric":[81],"regression":[82],"machine":[84,326],"learning":[85,281,327],"algorithms),":[86],"hyperparameter":[88,273,323],"tuning.":[89,324],"Methods":[90],"We":[91],"used":[92],"previously":[94],"published":[95],"HOPE":[96],"dataset":[97],"develop":[99],"a":[100,170,199,255],"for":[103,160,208],"anxiety":[104],"at":[105],"palliative":[106],"care":[107],"admission,":[108],"incorporating":[109],"34":[110],"candidate":[111],"predictors.":[112],"To":[113],"examine":[114],"impact":[116],"we":[120],"generated":[121],"seven":[122],"datasets":[123,141],"varying":[125],"sizes":[126,191],"through":[127],"random":[128,149],"sampling.":[129],"A":[130],"total":[131],"21":[133],"scenarios":[135],"were":[136,154,192,283],"evaluated":[137,168],"by":[138],"combining":[139],"these":[140],"approaches:":[145],"LASSO":[146,222,246],"logistic":[147],"regression,":[148],"forest,":[150],"XGBoost.":[152],"tuned":[155],"using":[156,169],"cross-validation":[157],"assessed":[159],"discrimination":[161,244],"(AUC)":[162],"calibration.":[164],"Prediction":[165],"was":[167],"bootstrap-based":[171,299],"framework,":[172],"including":[173],"mean":[175],"absolute":[176],"error":[178],"(MAPE)":[179],"related":[181],"instability":[182],"plots.":[183],"Results":[184],"In":[185],"this":[186],"dataset,":[188],"associated":[193,284],"lower":[195,225,256,286,290],"average":[196,253],"MAPE":[197,205,261,287],"higher":[200,243,252],"proportion":[201,257],"\u2264":[206],"5%":[207],"LASSO,":[209],"limited":[211],"additional":[212],"changes":[213],"beyond":[214],"reference":[216],"(":[218],"n":[219],"=":[220],"3,875).":[221],"also":[223,318],"showed":[224,242],"AUC":[226],"optimism":[227],"calibration":[229,264],"slopes":[230,265],"closer":[231],"1":[233,268],"across":[234,269],"increasing":[235],"sizes.":[237,271],"Random":[238],"forest":[239],"XGBoost":[241],"than":[245],"some":[248],"scenarios,":[249],"but":[250,288,317],"had":[251],"MAPE,":[254],"\u22645%,":[262],"further":[266],"from":[267],"Conservative":[272],"settings,":[274],"such":[275],"shallower":[277],"trees":[278],"slower":[280],"rates,":[282],"slightly":[289],"discrimination.":[291],"Conclusions":[292],"Our":[293],"results":[294],"provide":[295],"empirical":[296],"insights":[297],"into":[298],"Model":[307],"appeared":[309],"depend":[311],"not":[312],"only":[313],"on":[314,319],"approach":[321],"For":[325],"models,":[328],"considering":[329],"alongside":[331],"marginal":[332],"gains":[333],"accuracy":[335],"may":[336],"be":[337],"important.":[338],"Further":[339],"research":[340],"should":[341],"explore":[342],"alternative":[343],"validation":[344],"strategies":[345],"optimal":[347],"tuning":[348],"practices":[349],"guide":[351],"robust":[352]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-11-07T00:00:00"}