{"id":"https://openalex.org/W2117966604","doi":"https://doi.org/10.1136/jamia.2002.0090001","title":"Reference Standards, Judges, and Comparison Subjects: Roles for Experts in Evaluating System Performance","display_name":"Reference Standards, Judges, and Comparison Subjects: Roles for Experts in Evaluating System Performance","publication_year":2002,"publication_date":"2002-01-01","ids":{"openalex":"https://openalex.org/W2117966604","doi":"https://doi.org/10.1136/jamia.2002.0090001","mag":"2117966604","pmid":"https://pubmed.ncbi.nlm.nih.gov/11751799"},"language":"en","primary_location":{"id":"doi:10.1136/jamia.2002.0090001","is_oa":true,"landing_page_url":"https://doi.org/10.1136/jamia.2002.0090001","pdf_url":"https://academic.oup.com/jamia/article-pdf/9/1/1/2004184/9-1-1.pdf","source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://academic.oup.com/jamia/article-pdf/9/1/1/2004184/9-1-1.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053670656","display_name":"George Hripcsak","orcid":"https://orcid.org/0000-0003-2664-7614"},"institutions":[{"id":"https://openalex.org/I2799503643","display_name":"Columbia University Irving Medical Center","ror":"https://ror.org/01esghr10","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I2799503643"]},{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"G. Hripcsak","raw_affiliation_strings":["Department of Medical Informatics, Columbia University, New York, New York 10032, USA. hripcsak@columbia.edu","Affiliation of the authors: Columbia University, New York, New York"],"affiliations":[{"raw_affiliation_string":"Department of Medical Informatics, Columbia University, New York, New York 10032, USA. hripcsak@columbia.edu","institution_ids":["https://openalex.org/I2799503643","https://openalex.org/I78577930"]},{"raw_affiliation_string":"Affiliation of the authors: Columbia University, New York, New York","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064822514","display_name":"Adam Wilcox","orcid":"https://orcid.org/0000-0002-6305-735X"},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"A. Wilcox","raw_affiliation_strings":["Affiliation of the authors: Columbia University, New York, New York"],"affiliations":[{"raw_affiliation_string":"Affiliation of the authors: Columbia University, New York, New York","institution_ids":["https://openalex.org/I78577930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5053670656"],"corresponding_institution_ids":["https://openalex.org/I2799503643","https://openalex.org/I78577930"],"apc_list":{"value":3967,"currency":"USD","value_usd":3967},"apc_paid":null,"fwci":8.7973,"has_fulltext":true,"cited_by_count":90,"citation_normalized_percentile":{"value":0.97847292,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"9","issue":"1","first_page":"1","last_page":"15"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9847999811172485,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.984000027179718,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7832298278808594},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.596169114112854},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5195253491401672},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.477005273103714},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3468059003353119},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.32753995060920715}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7832298278808594},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.596169114112854},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5195253491401672},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.477005273103714},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3468059003353119},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.32753995060920715},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[{"descriptor_ui":"D005069","descriptor_name":"Evaluation Studies as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D005069","descriptor_name":"Evaluation Studies as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D005069","descriptor_name":"Evaluation Studies as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D007256","descriptor_name":"Information Systems","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":false},{"descriptor_ui":"D007256","descriptor_name":"Information Systems","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":false},{"descriptor_ui":"D007256","descriptor_name":"Information Systems","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":false},{"descriptor_ui":"D008722","descriptor_name":"Methods","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008722","descriptor_name":"Methods","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008722","descriptor_name":"Methods","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012015","descriptor_name":"Reference Standards","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012015","descriptor_name":"Reference Standards","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012015","descriptor_name":"Reference Standards","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":3,"locations":[{"id":"doi:10.1136/jamia.2002.0090001","is_oa":true,"landing_page_url":"https://doi.org/10.1136/jamia.2002.0090001","pdf_url":"https://academic.oup.com/jamia/article-pdf/9/1/1/2004184/9-1-1.pdf","source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},{"id":"pmid:11751799","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/11751799","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association : JAMIA","raw_type":null},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.199.7581","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.199.7581","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www2.dmi.columbia.edu/edu/G4060/readings-f03/week3-2002_JAMIA_Hripcsak_evalexpert.pdf","raw_type":"text"}],"best_oa_location":{"id":"doi:10.1136/jamia.2002.0090001","is_oa":true,"landing_page_url":"https://doi.org/10.1136/jamia.2002.0090001","pdf_url":"https://academic.oup.com/jamia/article-pdf/9/1/1/2004184/9-1-1.pdf","source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6541979941","display_name":null,"funder_award_id":"LM06910","funder_id":"https://openalex.org/F4320337372","funder_display_name":"U.S. National Library of Medicine"},{"id":"https://openalex.org/G7116188982","display_name":null,"funder_award_id":"grants","funder_id":"https://openalex.org/F4320307765","funder_display_name":"Pfizer"},{"id":"https://openalex.org/G7122601629","display_name":null,"funder_award_id":"grant","funder_id":"https://openalex.org/F4320307765","funder_display_name":"Pfizer"}],"funders":[{"id":"https://openalex.org/F4320307765","display_name":"Pfizer","ror":"https://ror.org/01xdqrp08"},{"id":"https://openalex.org/F4320337372","display_name":"U.S. National Library of Medicine","ror":"https://ror.org/0060t0j89"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2117966604.pdf","grobid_xml":"https://content.openalex.org/works/W2117966604.grobid-xml"},"referenced_works_count":90,"referenced_works":["https://openalex.org/W9230873","https://openalex.org/W23040107","https://openalex.org/W31448004","https://openalex.org/W144473711","https://openalex.org/W749720113","https://openalex.org/W1445338536","https://openalex.org/W1480687950","https://openalex.org/W1483001745","https://openalex.org/W1524272520","https://openalex.org/W1549872659","https://openalex.org/W1562956148","https://openalex.org/W1593089476","https://openalex.org/W1601162575","https://openalex.org/W1723317842","https://openalex.org/W1740869759","https://openalex.org/W1815059227","https://openalex.org/W1837244729","https://openalex.org/W1841816101","https://openalex.org/W1956220243","https://openalex.org/W1964100346","https://openalex.org/W1965380368","https://openalex.org/W1968114652","https://openalex.org/W1968119712","https://openalex.org/W1971644320","https://openalex.org/W1972182967","https://openalex.org/W1984431881","https://openalex.org/W1990748933","https://openalex.org/W1995221561","https://openalex.org/W1996844454","https://openalex.org/W1998647237","https://openalex.org/W1998735789","https://openalex.org/W2012301002","https://openalex.org/W2017086716","https://openalex.org/W2020594744","https://openalex.org/W2027528363","https://openalex.org/W2028548753","https://openalex.org/W2044702943","https://openalex.org/W2044742883","https://openalex.org/W2045917085","https://openalex.org/W2050460112","https://openalex.org/W2054996896","https://openalex.org/W2062884568","https://openalex.org/W2068335288","https://openalex.org/W2069659651","https://openalex.org/W2074706689","https://openalex.org/W2084139018","https://openalex.org/W2089428797","https://openalex.org/W2100899314","https://openalex.org/W2119265140","https://openalex.org/W2122164289","https://openalex.org/W2125726375","https://openalex.org/W2131819133","https://openalex.org/W2132515539","https://openalex.org/W2133843798","https://openalex.org/W2135417100","https://openalex.org/W2135696570","https://openalex.org/W2138031922","https://openalex.org/W2138501810","https://openalex.org/W2139372732","https://openalex.org/W2141639820","https://openalex.org/W2146980252","https://openalex.org/W2148930725","https://openalex.org/W2150290224","https://openalex.org/W2153804780","https://openalex.org/W2154104658","https://openalex.org/W2155491802","https://openalex.org/W2155524176","https://openalex.org/W2157825442","https://openalex.org/W2161676054","https://openalex.org/W2167945625","https://openalex.org/W2171695133","https://openalex.org/W2277039269","https://openalex.org/W2278901959","https://openalex.org/W2281651263","https://openalex.org/W2300138637","https://openalex.org/W2305896620","https://openalex.org/W2316990564","https://openalex.org/W2339530503","https://openalex.org/W2339530632","https://openalex.org/W2339926369","https://openalex.org/W2410297081","https://openalex.org/W2427199288","https://openalex.org/W3005505736","https://openalex.org/W3023003447","https://openalex.org/W3026755508","https://openalex.org/W4243654166","https://openalex.org/W4254165724","https://openalex.org/W4293168481","https://openalex.org/W4299374713","https://openalex.org/W4300722386"],"related_works":["https://openalex.org/W1585007175","https://openalex.org/W2382521049","https://openalex.org/W2144385241","https://openalex.org/W2165950148","https://openalex.org/W4253593777","https://openalex.org/W2951497643","https://openalex.org/W4300101996","https://openalex.org/W3101625811","https://openalex.org/W2338854850","https://openalex.org/W2184239527"],"abstract_inverted_index":{"Medical":[0],"informatics":[1],"systems":[2,20],"are":[3,127],"often":[4,22],"designed":[5],"to":[6,96],"perform":[7],"at":[8],"the":[9,16,31,56,71,83,87,105,121,147],"level":[10],"of":[11,15,18,26,73,107,140,149],"human":[12],"experts.":[13,76],"Evaluation":[14],"performance":[17,46,57],"these":[19],"is":[21,34,51,58],"constrained":[23],"by":[24],"lack":[25],"reference":[27,99],"standards,":[28],"either":[29],"because":[30,38],"appropriate":[32,41],"response":[33,42],"not":[35,52],"known":[36],"or":[37,60],"no":[39],"simple":[40],"exists.":[43],"Even":[44],"when":[45],"can":[47,64,80,93,103,114,123],"be":[48,65,94,124],"assessed,":[49],"it":[50],"always":[53],"clear":[54],"whether":[55],"sufficient":[59],"reasonable.":[61],"These":[62,126],"challenges":[63],"addressed":[66],"if":[67],"an":[68],"evaluator":[69],"enlists":[70],"help":[72,145],"clinical":[74],"domain":[75],"1)":[77],"The":[78,112],"experts":[79,102,113,150],"carry":[81],"out":[82],"same":[84],"tasks":[85],"as":[86,116],"system,":[88],"and":[89,138,142],"then":[90],"their":[91],"responses":[92],"combined":[95],"generate":[97],"a":[98],"standard.":[100],"2)The":[101],"judge":[104],"appropriateness":[106],"system":[108,122],"output":[109],"directly.":[110],"3)":[111],"serve":[115],"comparison":[117],"subjects":[118],"with":[119],"which":[120],"compared.":[125],"separate":[128],"roles":[129,148],"that":[130],"have":[131],"different":[132],"implications":[133],"for":[134],"study":[135,153],"design,":[136],"metrics,":[137],"issues":[139],"reliability":[141],"validity.":[143],"Diagrams":[144],"delineate":[146],"in":[151],"complex":[152],"designs.":[154]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":4},{"year":2016,"cited_by_count":2},{"year":2015,"cited_by_count":2},{"year":2014,"cited_by_count":4},{"year":2013,"cited_by_count":6},{"year":2012,"cited_by_count":4}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}