{"id":"https://openalex.org/W4241118407","doi":"https://doi.org/10.7326/0003-4819-150-6-200903170-00022","title":"Are Risk Stratification Tables the Best Way to Evaluate Model Performance?","display_name":"Are Risk Stratification Tables the Best Way to Evaluate Model Performance?","publication_year":2009,"publication_date":"2009-03-17","ids":{"openalex":"https://openalex.org/W4241118407","doi":"https://doi.org/10.7326/0003-4819-150-6-200903170-00022"},"language":"en","primary_location":{"id":"doi:10.7326/0003-4819-150-6-200903170-00022","doi":"https://doi.org/10.7326/0003-4819-150-6-200903170-00022","is_oa":false,"landing_page_url":"https://doi.org/10.7326/0003-4819-150-6-200903170-00022","pdf_url":null,"source":{"id":"https://openalex.org/S119722071","display_name":"Annals of Internal Medicine","issn_l":"0003-4819","issn":["0003-4819","1539-3704"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","dongbi-a","doyens","jufo-3","ki-jl-3","medline","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310316812","host_organization_name":"American College of Physicians","host_organization_lineage":["https://openalex.org/P4310316812"],"host_organization_lineage_names":["American College of Physicians"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Annals of Internal Medicine","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050864343","display_name":"Holly Janes","orcid":"https://orcid.org/0000-0002-3237-984X","observed_orcids":["https://orcid.org/0000-0002-3237-984X"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210089486","display_name":"Fred Hutch Cancer Center","ror":"https://ror.org/007ps6h72","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210089486"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Holly Janes","raw_affiliation_strings":["From Fred Hutchinson Cancer Research Center, Seattle, WA 98109, and University of Washington, Seattle, WA 98195"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"From Fred Hutchinson Cancer Research Center, Seattle, WA 98109, and University of Washington, Seattle, WA 98195","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I4210089486"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Margaret S. Pepe","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210089486","display_name":"Fred Hutch Cancer Center","ror":"https://ror.org/007ps6h72","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210089486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Margaret S. Pepe","raw_affiliation_strings":["From Fred Hutchinson Cancer Research Center, Seattle, WA 98109, and University of Washington, Seattle, WA 98195"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"From Fred Hutchinson Cancer Research Center, Seattle, WA 98109, and University of Washington, Seattle, WA 98195","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I4210089486"]}]},{"author_position":"last","author":{"id":null,"display_name":"Wen Gu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210089486","display_name":"Fred Hutch Cancer Center","ror":"https://ror.org/007ps6h72","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210089486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wen Gu","raw_affiliation_strings":["From Fred Hutchinson Cancer Research Center, Seattle, WA 98109, and University of Washington, Seattle, WA 98195"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"From Fred Hutchinson Cancer Research Center, Seattle, WA 98109, and University of Washington, Seattle, WA 98195","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I4210089486"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5050864343"],"corresponding_institution_ids":["https://openalex.org/I201448701","https://openalex.org/I4210089486"],"apc_list":null,"apc_paid":null,"fwci":0.2565,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.67112533,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"150","issue":"6","first_page":"428","last_page":"428"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14374","display_name":"Statistical Methods in Epidemiology","score":0.9603999853134155,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14374","display_name":"Statistical Methods in Epidemiology","score":0.9603999853134155,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.01209999993443489,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.003599999938160181,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/model-evaluation","display_name":"model evaluation","score":0.9340000152587891},{"id":"https://openalex.org/keywords/risk-allocation","display_name":"risk allocation","score":0.578000009059906},{"id":"https://openalex.org/keywords/cancer-risk-prediction","display_name":"cancer risk prediction","score":0.44699999690055847},{"id":"https://openalex.org/keywords/prognostic-model","display_name":"prognostic model","score":0.4959999918937683}],"concepts":[{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.6205596327781677},{"id":"https://openalex.org/C3020404979","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk stratification","level":2,"score":0.5173665881156921},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.4294520318508148},{"id":"https://openalex.org/C161191863","wikidata":"https://www.wikidata.org/wiki/Q199655","display_name":"Library science","level":1,"score":0.3691198527812958},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.1300700604915619},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.10086807608604431},{"id":"https://openalex.org/C99454951","wikidata":"https://www.wikidata.org/wiki/Q932068","display_name":"Environmental health","level":1,"score":0.0988057553768158}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.7326/0003-4819-150-6-200903170-00022","doi":"https://doi.org/10.7326/0003-4819-150-6-200903170-00022","is_oa":false,"landing_page_url":"https://doi.org/10.7326/0003-4819-150-6-200903170-00022","pdf_url":null,"source":{"id":"https://openalex.org/S119722071","display_name":"Annals of Internal Medicine","issn_l":"0003-4819","issn":["0003-4819","1539-3704"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","dongbi-a","doyens","jufo-3","ki-jl-3","medline","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310316812","host_organization_name":"American College of Physicians","host_organization_lineage":["https://openalex.org/P4310316812"],"host_organization_lineage_names":["American College of Physicians"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Annals of Internal Medicine","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being","score":0.4000000059604645}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":3,"referenced_works":["https://openalex.org/W2108284772","https://openalex.org/W2133977873","https://openalex.org/W2151680906"],"related_works":["https://openalex.org/W2143717212","https://openalex.org/W7096541364","https://openalex.org/W2563274022","https://openalex.org/W7095805280","https://openalex.org/W7163403615","https://openalex.org/W2151680906","https://openalex.org/W2464987977","https://openalex.org/W4416548477","https://openalex.org/W3111143494","https://openalex.org/W1504852314"],"abstract_inverted_index":{"Letters17":[0],"March":[1,485,499,504],"2009Are":[2],"Risk":[3,436,440,454],"Stratification":[4,441,455],"Tables":[5,442,456],"the":[6,122,132,138,149,156,171,181,188,197,223,250,279,302,338,360,371,433,457],"Best":[7,458],"Way":[8,459],"to":[9,103,187,204,374,460],"Evaluate":[10,461],"Model":[11,462],"Performance?Holly":[12],"Janes,":[13,23,402],"PhD,":[14,18],"Margaret":[15,46,404,446],"S.":[16,47,405,447],"Pepe,":[17,48,406],"and":[19,33,58,71,83,97,113,159,276,307,376,398,420,450,467],"Wen":[20,72,408,451],"Gu,":[21,73,409],"MSHolly":[22],"PhDFrom":[24,49],"Fred":[25,50,75,412],"Hutchinson":[26,51,76,413],"Cancer":[27,52,77,319,414],"Research":[28,53,78,415],"Center,":[29,54,79,416],"Seattle,":[30,37,55,62,80,87,417,424],"WA":[31,38,56,63,81,88,418,425],"98109,":[32,57,82,419],"University":[34,59,84,421],"of":[35,60,85,124,134,173,184,200,225,252,278,304,313,340,364,422,435,474,478,491,514],"Washington,":[36,61,86,423],"98195.Search":[39,64,89],"for":[40,65,90,115,121,144,164,310,336,476],"more":[41,66,91],"papers":[42,67,92],"by":[43,68,93,137,192,269,378,438,511],"this":[44,69,94,152],"author,":[45,70],"MSFrom":[74],"authorAuthor,":[95],"Article,":[96,397],"Disclosure":[98,399],"Informationhttps://doi.org/10.7326/0003-4819-150-6-200903170-00022":[99],"SectionsAboutFull":[100],"TextPDF":[101],"ToolsAdd":[102],"favoritesDownload":[104],"CitationsTrack":[105],"CitationsPermissions":[106],"ShareFacebookTwitterLinkedInRedditEmail":[107],"IN":[108],"RESPONSE:We":[109],"thank":[110],"Drs.":[111],"Stern":[112,466],"Smith":[114,470],"their":[116],"thoughts":[117],"on":[118,358],"improving":[119],"methods":[120],"evaluation":[123,483],"risk":[125,139,175,185,285,312],"prediction":[126,140],"models.":[127],"We":[128],"agree":[129],"wholeheartedly":[130],"that":[131,206],"distribution":[133,183],"risks":[135],"predicted":[136],"model":[141,146,333],"is":[142,203],"key":[143],"evaluating":[145,337],"performance.":[147],"In":[148,169],"statistical":[150],"literature,":[151],"has":[153],"been":[154],"called":[155],"predictiveness":[157,224,251,339],"curve,":[158],"we":[160],"have":[161],"advocated":[162],"strongly":[163],"its":[165,229],"use":[166],"(1,":[167],"2).":[168],"fact,":[170],"margins":[172],"a":[174,226,232,253,365],"stratification":[176],"table":[177],"display":[178],"exactly":[179],"this:":[180],"population":[182],"according":[186],"2":[189],"models,":[190],"albeit":[191],"using":[193],"discrete":[194],"categories.":[195],"Because":[196],"main":[198],"goal":[199],"our":[201],"article":[202],"emphasize":[205],"...References1.":[207],"Pepe":[208,245,262,270,297,328,351,448],"MS,":[209,263,298,352],"Feng":[210,247,353],"Z,":[211,354],"Huang":[212,242,326],"Y,":[213,243,327],"Longton":[214],"G,":[215],"Prentice":[216],"R,":[217],"Thompson":[218],"IM,":[219],"et":[220,271,382],"al.":[221],"Integrating":[222],"marker":[227],"with":[228],"performance":[230,303],"as":[231],"classifier.":[233],"Am":[234],"J":[235,317],"Epidemiol.":[236],"2008;167:362-8.":[237],"[PMID:":[238,258,293,322,392],"17982157]":[239],"CrossrefMedlineGoogle":[240,260,324],"Scholar2.":[241],"Sullivan":[244],"M,":[246],"Z.":[248],"Evaluating":[249],"continuous":[254,341],"marker.":[255],"Biometrics.":[256,346],"2007;63:1181-8.":[257],"17489968]":[259],"Scholar3.":[261],"Janes":[264,299,444],"H,":[265],"Gu":[266,355,452],"JW.":[267,356],"Letter":[268],"al":[272],"regarding":[273],"article,":[274],"\u201cUse":[275],"misuse":[277],"receiver":[280],"operating":[281,493],"characteristic":[282,494],"curve":[283,373],"in":[284,343,385],"prediction\u201d":[286],"[Letter].":[287],"Circulation.":[288],"2007;116:e132;":[289],"author":[290],"reply":[291],"e134.":[292],"17679623]":[294],"Google":[295,349,394],"Scholar4.":[296],"HE.":[300],"Gauging":[301],"SNPs,":[305],"biomarkers,":[306],"clinical":[308],"factors":[309],"predicting":[311],"breast":[314],"cancer":[315],"[Editorial].":[316],"Natl":[318],"Inst.":[320],"2008;100:978-9.":[321],"18612128]":[323],"Scholar5.":[325],"MS.":[329],"A":[330],"parametric":[331],"ROC":[332,372],"based":[334],"approach":[335],"markers":[342],"case-control":[344],"studies.":[345],"2009.":[347],"[Forthcoming].":[348],"Scholar6.":[350],"Comments":[357],"\u2018Evaluating":[359],"added":[361],"predictive":[362],"ability":[363],"new":[366],"marker:":[367],"From":[368,411],"area":[369],"under":[370],"reclassification":[375,475],"beyond\u2019":[377],"M.":[379],"J.":[380],"Pencina":[381],"al.,":[383],"Statistics":[384],"Medicine":[386],"(DOI:":[387],"10.1002/sim.2929).":[388],"Stat":[389],"Med.":[390],"2008;27:173-81.":[391],"17671958]":[393],"Scholar":[395],"Author,":[396],"InformationAuthors:":[400],"Holly":[401,443],"PhD;":[403,407],"MSAffiliations:":[410],"98195.Disclosures:":[426],"None":[427],"disclosed.":[428],"PreviousarticleNextarticle":[429],"Advertisement":[430],"FiguresReferencesRelatedDetailsSee":[431],"AlsoAssessing":[432],"Value":[434],"Predictions":[437],"Using":[439],",":[445,449],"Are":[453],"Performance?":[463],"Ralph":[464],"H.":[465],"Dean":[468],"E.":[469],"Metrics":[471],"Cited":[472],"byUse":[473],"assessment":[477],"improved":[479],"prediction:":[480],"an":[481],"empirical":[482],"17":[484,498,503],"2009Volume":[486],"150,":[487],"Issue":[488,501],"6Page:":[489],"428KeywordsCellsConflicts":[490],"interestReceiver":[492],"curveStatistical":[495],"distributions":[496],"ePublished:":[497],"2009":[500,505,510],"Published:":[502],"Copyright":[506],"&":[507],"PermissionsCopyright":[508],"\u00a9":[509],"American":[512],"College":[513],"Physicians.":[515],"All":[516],"Rights":[517],"Reserved.PDF":[518],"downloadLoading":[519],"...":[520]},"counts_by_year":[],"updated_date":"2026-10-14T16:39:01.820549","created_date":"2022-05-12T00:00:00"}