{"id":"https://openalex.org/W2147860335","doi":"https://doi.org/10.1136/jamia.2002.0090087","title":"Reference Standards in Evaluating System Performance","display_name":"Reference Standards in Evaluating System Performance","publication_year":2002,"publication_date":"2002-01-01","ids":{"openalex":"https://openalex.org/W2147860335","doi":"https://doi.org/10.1136/jamia.2002.0090087","mag":"2147860335","pmid":"https://pubmed.ncbi.nlm.nih.gov/11751807"},"language":"en","primary_location":{"id":"doi:10.1136/jamia.2002.0090087","is_oa":true,"landing_page_url":"https://doi.org/10.1136/jamia.2002.0090087","pdf_url":"https://academic.oup.com/jamia/article-pdf/9/1/87/2012439/9-1-87.pdf","source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://academic.oup.com/jamia/article-pdf/9/1/87/2012439/9-1-87.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091300942","display_name":"Ronald Miller","orcid":null},"institutions":[{"id":"https://openalex.org/I901861585","display_name":"Vanderbilt University Medical Center","ror":"https://ror.org/05dq2gs74","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210162197","https://openalex.org/I901861585"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"R. A. Miller","raw_affiliation_strings":["Affiliation of the author: Informatics Center, Vanderbilt University Medical Center, Nashville, Tennessee"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Affiliation of the author: Informatics Center, Vanderbilt University Medical Center, Nashville, Tennessee","institution_ids":["https://openalex.org/I901861585"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5091300942"],"corresponding_institution_ids":["https://openalex.org/I901861585"],"apc_list":{"value":3967,"currency":"USD","value_usd":3967},"apc_paid":null,"fwci":4.705,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.95358025,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"9","issue":"1","first_page":"87","last_page":"88"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10350","display_name":"Electronic Health Records Systems","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/3605","display_name":"Health Information Management"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T10350","display_name":"Electronic Health Records Systems","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/3605","display_name":"Health Information Management"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9886999726295471,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9627000093460083,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7574267387390137},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.7537543773651123},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6039055585861206},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5503990054130554},{"id":"https://openalex.org/keywords/informatics","display_name":"Informatics","score":0.45683687925338745},{"id":"https://openalex.org/keywords/event","display_name":"Event (particle physics)","score":0.43358609080314636},{"id":"https://openalex.org/keywords/health-informatics","display_name":"Health informatics","score":0.4303607940673828},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.35942408442497253},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.32118338346481323},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.13669052720069885}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7574267387390137},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7537543773651123},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6039055585861206},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5503990054130554},{"id":"https://openalex.org/C191630685","wikidata":"https://www.wikidata.org/wiki/Q4027615","display_name":"Informatics","level":2,"score":0.45683687925338745},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.43358609080314636},{"id":"https://openalex.org/C145642194","wikidata":"https://www.wikidata.org/wiki/Q870895","display_name":"Health informatics","level":3,"score":0.4303607940673828},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.35942408442497253},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.32118338346481323},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.13669052720069885},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C159110408","wikidata":"https://www.wikidata.org/wiki/Q121176","display_name":"Nursing","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138816342","wikidata":"https://www.wikidata.org/wiki/Q189603","display_name":"Public health","level":2,"score":0.0}],"mesh":[{"descriptor_ui":"D005069","descriptor_name":"Evaluation Studies as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D005069","descriptor_name":"Evaluation Studies as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D005069","descriptor_name":"Evaluation Studies as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D007256","descriptor_name":"Information Systems","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":false},{"descriptor_ui":"D007256","descriptor_name":"Information Systems","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":false},{"descriptor_ui":"D007256","descriptor_name":"Information Systems","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":false},{"descriptor_ui":"D008722","descriptor_name":"Methods","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008722","descriptor_name":"Methods","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008722","descriptor_name":"Methods","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012015","descriptor_name":"Reference Standards","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012015","descriptor_name":"Reference Standards","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012015","descriptor_name":"Reference Standards","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":2,"locations":[{"id":"doi:10.1136/jamia.2002.0090087","is_oa":true,"landing_page_url":"https://doi.org/10.1136/jamia.2002.0090087","pdf_url":"https://academic.oup.com/jamia/article-pdf/9/1/87/2012439/9-1-87.pdf","source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},{"id":"pmid:11751807","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/11751807","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association : JAMIA","raw_type":null}],"best_oa_location":{"id":"doi:10.1136/jamia.2002.0090087","is_oa":true,"landing_page_url":"https://doi.org/10.1136/jamia.2002.0090087","pdf_url":"https://academic.oup.com/jamia/article-pdf/9/1/87/2012439/9-1-87.pdf","source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2147860335.pdf"},"referenced_works_count":5,"referenced_works":["https://openalex.org/W1577496953","https://openalex.org/W1841012535","https://openalex.org/W1968013239","https://openalex.org/W2117966604","https://openalex.org/W2146980252"],"related_works":["https://openalex.org/W2397677514","https://openalex.org/W1495820741","https://openalex.org/W2914293784","https://openalex.org/W2990177913","https://openalex.org/W1451471413","https://openalex.org/W2138729887","https://openalex.org/W2401910071","https://openalex.org/W4323056170","https://openalex.org/W2126732312","https://openalex.org/W2015849715"],"abstract_inverted_index":{"The":[0,90,170],"paper":[1],"in":[2,18,65,101],"this":[3,73],"issue":[4],"by":[5],"Hripcsak":[6,56],"and":[7,12,25,57,126,135,165],"Wilcox,":[8],"\u201cReference":[9],"Standards,":[10],"Judges,":[11],"Comparison":[13],"Subjects:":[14],"Roles":[15],"for":[16],"Experts":[17],"Evaluating":[19],"System":[20],"Performance,\u201d1":[21],"is":[22,39,183],"well":[23],"written":[24],"presents":[26],"a":[27,87,115,192,202,212],"thoughtful":[28],"analysis":[29],"of":[30,44,86,121,137,150,173,201,208],"the":[31,34,42,77,178,181,186,198,209,219],"topic.":[32],"As":[33],"authors":[35],"acknowledge,":[36],"however,":[37],"there":[38],"more":[40],"to":[41,54,62,157,191],"evaluation":[43,100,174,200],"clinical":[45,167],"informatics":[46,103],"systems":[47,67,125,148],"than":[48,96,185,221],"can":[49],"be":[50,176],"accomplished":[51],"through":[52],"comparison":[53],"experts.2,3":[55],"Wilcox":[58],"focus":[59],"on":[60,76,205],"\u201chow":[61],"use":[63,82,218],"experts":[64,83],"evaluating":[66],"when":[68,153,216,222],"one":[69,81],"needs":[70],"them,\u201d":[71],"whereas":[72],"commentary":[74],"focuses":[75],"question,":[78],"\u201cwhen":[79],"should":[80,104,175],"as":[84,107,114],"part":[85],"system's":[88],"evaluation\u201d":[89],"two":[91],"perspectives":[92],"are":[93,149],"complementary":[94],"rather":[95],"contradictory.\n\nAs":[97],"noted":[98],"previously,4\n\nSystem":[99],"biomedical":[102],"take":[105],"place":[106],"an":[108],"ongoing,":[109],"strategically":[110],"planned":[111],"process,":[112],"not":[113,162],"single":[116],"event":[117],"or":[118,195],"small":[119],"number":[120],"episodes.":[122],"Complex":[123],"software":[124],"accepted":[127],"medical":[128],"practices":[129],"both":[130],"evolve":[131],"rapidly,":[132],"so":[133],"evaluators":[134],"readers":[136],"evaluations":[138],"face":[139],"moving":[140],"targets.":[141],"\u2026":[142,225],"[C]urrent":[143],"thinking":[144],"recognizes":[145],"that":[146],"such":[147],"value":[151],"only":[152],"they":[154,217,223],"help":[155],"users":[156,207],"solve":[158,166],"users'":[159],"problems.":[160,169],"Users,":[161],"systems,":[163],"characterize":[164],"diagnostic":[168],"ultimate":[171,199],"unit":[172],"whether":[177,206],"user":[179,188],"plus":[180],"system":[182,203,210,220],"better":[184,215],"unaided":[187],"with":[189],"respect":[190],"specified":[193,213],"task":[194,214],"problem.\u2026":[196],"\n\nIf":[197],"depends":[204],"perform":[211],"don't,":[224]},"counts_by_year":[{"year":2018,"cited_by_count":1},{"year":2012,"cited_by_count":1}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}