{"id":"https://openalex.org/W2154745347","doi":"https://doi.org/10.1109/titb.2008.2009440","title":"Indexes for Three-Class Classification Performance Assessment\u2014An Empirical Comparison","display_name":"Indexes for Three-Class Classification Performance Assessment\u2014An Empirical Comparison","publication_year":2009,"publication_date":"2009-01-20","ids":{"openalex":"https://openalex.org/W2154745347","doi":"https://doi.org/10.1109/titb.2008.2009440","mag":"2154745347","pmid":"https://pubmed.ncbi.nlm.nih.gov/19171528"},"language":"en","primary_location":{"id":"doi:10.1109/titb.2008.2009440","is_oa":false,"landing_page_url":"https://doi.org/10.1109/titb.2008.2009440","pdf_url":null,"source":{"id":"https://openalex.org/S68937846","display_name":"IEEE Transactions on Information Technology in Biomedicine","issn_l":"1089-7771","issn":["1089-7771","1558-0032"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Information Technology in Biomedicine","raw_type":"journal-article"},"type":"paratext","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026691023","display_name":"Mehul Sampat","orcid":null},"institutions":[{"id":"https://openalex.org/I1283280774","display_name":"Brigham and Women's Hospital","ror":"https://ror.org/04b6nzv94","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283280774","https://openalex.org/I48633490"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"M.P. Sampat","raw_affiliation_strings":["Center for Neurological Imaging, Department of Radiology, Brigham and Women's Hospital, Boston, MA 02115, USA. mehul.sampat@ieee.org","Dept. of Radiol., Brigham and Women's Hosp., Boston, MA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Neurological Imaging, Department of Radiology, Brigham and Women's Hospital, Boston, MA 02115, USA. mehul.sampat@ieee.org","institution_ids":["https://openalex.org/I1283280774"]},{"raw_affiliation_string":"Dept. of Radiol., Brigham and Women's Hosp., Boston, MA","institution_ids":["https://openalex.org/I1283280774"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101494867","display_name":"Ameet C. Patel","orcid":"https://orcid.org/0000-0002-2916-2754"},"institutions":[{"id":"https://openalex.org/I867280407","display_name":"The University of Texas Southwestern Medical Center","ror":"https://ror.org/05byvp690","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I867280407"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"A.C. Patel","raw_affiliation_strings":["University of Texas Southwestern, Dallas, TX, USA","University of Texas Southwestern, Dallas, TX,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Texas Southwestern, Dallas, TX, USA","institution_ids":["https://openalex.org/I867280407"]},{"raw_affiliation_string":"University of Texas Southwestern, Dallas, TX,","institution_ids":["https://openalex.org/I867280407"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045909014","display_name":"Yuhling Wang","orcid":"https://orcid.org/0000-0002-3090-5017"},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuhling Wang","raw_affiliation_strings":["University of Virginia, Charlottesville, VA, US","Dept. of Biomed. Eng., Univ. of Virginia, Charlottesville, VA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Virginia, Charlottesville, VA, US","institution_ids":["https://openalex.org/I51556381"]},{"raw_affiliation_string":"Dept. of Biomed. Eng., Univ. of Virginia, Charlottesville, VA","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062232268","display_name":"Sumit Gupta","orcid":"https://orcid.org/0000-0003-3193-7077"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"S. Gupta","raw_affiliation_strings":["University of Texas at Austin, Austin, TX, US","Dept. of Electr. & Comput. Eng., Univ. of Texas, Austin, TX#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Texas at Austin, Austin, TX, US","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"Dept. of Electr. & Comput. Eng., Univ. of Texas, Austin, TX#TAB#","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001655813","display_name":"Chih-Wen Kan","orcid":null},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chih-Wen Kan","raw_affiliation_strings":["University of Texas at Austin, Austin, TX, US","Dept. of Biomed. Eng., Univ. of Texas, Austin, TX"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Texas at Austin, Austin, TX, US","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"Dept. of Biomed. Eng., Univ. of Texas, Austin, TX","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075463806","display_name":"Alan C. Bovik","orcid":"https://orcid.org/0000-0001-6067-710X"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"A.C. Bovik","raw_affiliation_strings":["University of Texas at Austin, Austin, TX, US","Dept. of Electr. & Comput. Eng., Univ. of Texas, Austin, TX#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Texas at Austin, Austin, TX, US","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"Dept. of Electr. & Comput. Eng., Univ. of Texas, Austin, TX#TAB#","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062714971","display_name":"Mia K. Markey","orcid":"https://orcid.org/0000-0001-8186-4959"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"M.K. Markey","raw_affiliation_strings":["University of Texas at Austin, Austin, TX, US","Dept. of Biomed. Eng., Univ. of Texas, Austin, TX"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Texas at Austin, Austin, TX, US","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"Dept. of Biomed. Eng., Univ. of Texas, Austin, TX","institution_ids":["https://openalex.org/I86519309"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"13","issue":"3","first_page":"300","last_page":"312"},"is_retracted":false,"is_paratext":true,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11490","display_name":"Hydrological Forecasting Using AI","score":0.9386000037193298,"subfield":{"id":"https://openalex.org/subfields/2305","display_name":"Environmental Engineering"},"field":{"id":"https://openalex.org/fields/23","display_name":"Environmental Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11490","display_name":"Hydrological Forecasting Using AI","score":0.9386000037193298,"subfield":{"id":"https://openalex.org/subfields/2305","display_name":"Environmental Engineering"},"field":{"id":"https://openalex.org/fields/23","display_name":"Environmental Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.913100004196167,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.6158125996589661},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.44909578561782837},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3790472745895386},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.34318673610687256},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.3410167098045349}],"concepts":[{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.6158125996589661},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.44909578561782837},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3790472745895386},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34318673610687256},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3410167098045349}],"mesh":[{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D001943","descriptor_name":"Breast Neoplasms","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":false},{"descriptor_ui":"D001943","descriptor_name":"Breast Neoplasms","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":false},{"descriptor_ui":"D001943","descriptor_name":"Breast Neoplasms","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":false},{"descriptor_ui":"D003198","descriptor_name":"Computer Simulation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003198","descriptor_name":"Computer Simulation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003198","descriptor_name":"Computer Simulation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003627","descriptor_name":"Data Interpretation, Statistical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003627","descriptor_name":"Data Interpretation, Statistical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003627","descriptor_name":"Data Interpretation, Statistical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003658","descriptor_name":"Decision Making, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003658","descriptor_name":"Decision Making, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003658","descriptor_name":"Decision Making, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003936","descriptor_name":"Diagnosis, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003936","descriptor_name":"Diagnosis, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003936","descriptor_name":"Diagnosis, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D006258","descriptor_name":"Head and Neck Neoplasms","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":false},{"descriptor_ui":"D006258","descriptor_name":"Head and Neck Neoplasms","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":false},{"descriptor_ui":"D006258","descriptor_name":"Head and Neck Neoplasms","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008327","descriptor_name":"Mammography","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008327","descriptor_name":"Mammography","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008327","descriptor_name":"Mammography","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012372","descriptor_name":"ROC Curve","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012372","descriptor_name":"ROC Curve","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012372","descriptor_name":"ROC Curve","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D013057","descriptor_name":"Spectrum Analysis","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D013057","descriptor_name":"Spectrum Analysis","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D013057","descriptor_name":"Spectrum Analysis","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D019540","descriptor_name":"Area Under Curve","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D019540","descriptor_name":"Area Under Curve","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D019540","descriptor_name":"Area Under Curve","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D035264","descriptor_name":"Flowers","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":false},{"descriptor_ui":"D035264","descriptor_name":"Flowers","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":false},{"descriptor_ui":"D035264","descriptor_name":"Flowers","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":false}],"locations_count":3,"locations":[{"id":"doi:10.1109/titb.2008.2009440","is_oa":false,"landing_page_url":"https://doi.org/10.1109/titb.2008.2009440","pdf_url":null,"source":{"id":"https://openalex.org/S68937846","display_name":"IEEE Transactions on Information Technology in Biomedicine","issn_l":"1089-7771","issn":["1089-7771","1558-0032"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Information Technology in Biomedicine","raw_type":"journal-article"},{"id":"pmid:19171528","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/19171528","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on information technology in biomedicine : a publication of the IEEE Engineering in Medicine and Biology Society","raw_type":null},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.156.6966","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.156.6966","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://live.ece.utexas.edu/publications/2009/mps_tinfbiomed_may09.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W1484011530","https://openalex.org/W1535653168","https://openalex.org/W1912982817","https://openalex.org/W1968114652","https://openalex.org/W1981744949","https://openalex.org/W1996861208","https://openalex.org/W2001619934","https://openalex.org/W2033539961","https://openalex.org/W2038781708","https://openalex.org/W2048537487","https://openalex.org/W2049663356","https://openalex.org/W2061597353","https://openalex.org/W2062818725","https://openalex.org/W2087760654","https://openalex.org/W2094295261","https://openalex.org/W2111732446","https://openalex.org/W2113333142","https://openalex.org/W2124965119","https://openalex.org/W2139177051","https://openalex.org/W2139862720","https://openalex.org/W2142012730","https://openalex.org/W2143225045","https://openalex.org/W2156772309","https://openalex.org/W2157300764","https://openalex.org/W2161578928","https://openalex.org/W2168366176","https://openalex.org/W2324020016","https://openalex.org/W2328176404","https://openalex.org/W2799061466","https://openalex.org/W4244494905"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W1979597421","https://openalex.org/W2007980826","https://openalex.org/W4245490552","https://openalex.org/W2061531152","https://openalex.org/W3002753104","https://openalex.org/W2077600819","https://openalex.org/W1587224694","https://openalex.org/W2042127053","https://openalex.org/W2142036596"],"abstract_inverted_index":{"Assessment":[0],"of":[1,9,96,109,127,184,201,209,239,242],"classifier":[2,30,132,165,202],"performance":[3,60,95,133,170,203],"is":[4,112,173],"critical":[5],"for":[6,58,92,98,121],"fair":[7],"comparison":[8],"methods,":[10],"including":[11],"considering":[12],"alternative":[13],"models":[14],"or":[15],"parameters":[16],"during":[17],"system":[18],"design.":[19],"The":[20,107],"assessment":[21,56],"must":[22,34],"not":[23],"only":[24],"provide":[25,130,232],"meaningful":[26],"data":[27],"on":[28],"the":[29,94,136,163,179,185,191,197,207,218,234,240,243],"efficacy,":[31],"but":[32],"it":[33,172],"do":[35],"so":[36],"in":[37,151,211],"a":[38,52,131,212],"concise":[39,55],"and":[40,54,61,78,182,204,216,223,236],"clear":[41,53],"manner.":[42],"For":[43],"two-class":[44],"classification":[45,76,155,214,254],"problems,":[46,101],"receiver":[47],"operating":[48],"characteristic":[49],"analysis":[50],"provides":[51,196],"methodology":[57],"reporting":[59],"comparing":[62],"competing":[63],"systems.":[64],"However,":[65],"many":[66],"other":[67],"important":[68],"biomedical":[69],"questions":[70],"cannot":[71],"be":[72,248],"posed":[73],"as":[74,230],"\"two-class\"":[75],"tasks":[77],"more":[79],"than":[80],"two":[81,253],"classes":[82],"are":[83,143,157],"often":[84],"necessary.":[85],"While":[86],"several":[87],"methods":[88,116,129,219],"have":[89,118,226],"been":[90,104,119],"proposed":[91,120,193,220],"assessing":[93,122],"classifiers":[97],"such":[99],"multiclass":[100,123],"none":[102],"has":[103],"widely":[105],"accepted.":[106],"purpose":[108],"this":[110],"paper":[111],"to":[113,175,250],"critically":[114],"review":[115],"that":[117],"classifiers.":[124],"A":[125],"number":[126],"these":[128,160],"index":[134],"called":[135],"volume":[137],"under":[138],"surface":[139],"(VUS).":[140],"Empirical":[141],"comparisons":[142],"carried":[144],"out":[145],"using":[146,168],"4":[147],"three-class":[148],"case":[149],"studies,":[150],"which":[152],"three":[153],"popular":[154],"techniques":[156],"evaluated":[158],"with":[159],"methods.":[161],"Since":[162],"same":[164],"was":[166],"assessed":[167],"multiple":[169],"indexes,":[171],"possible":[174],"gain":[176],"insight":[177,205],"into":[178],"relative":[180],"strengths":[181],"weakness":[183],"measures.":[186],"We":[187],"conclude":[188],"that:":[189],"1)":[190],"method":[192],"by":[194,221],"Scurfield":[195],"most":[198],"detailed":[199],"description":[200],"about":[206],"sources":[208],"error":[210],"given":[213],"task":[215],"2)":[217],"He":[222],"Nakas":[224],"also":[225],"great":[227],"practical":[228],"utility":[229],"they":[231],"both":[233],"VUS":[235],"an":[237],"estimate":[238],"variance":[241],"VUS.":[244],"These":[245],"estimates":[246],"can":[247],"used":[249],"statistically":[251],"compare":[252],"algorithms.":[255]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":2},{"year":2017,"cited_by_count":2},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":2},{"year":2013,"cited_by_count":5}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}