{"id":"https://openalex.org/W2108420460","doi":"https://doi.org/10.1093/bib/bbq073","title":"An empirical assessment of validation practices for molecular classifiers","display_name":"An empirical assessment of validation practices for molecular classifiers","publication_year":2011,"publication_date":"2011-02-07","ids":{"openalex":"https://openalex.org/W2108420460","doi":"https://doi.org/10.1093/bib/bbq073","mag":"2108420460","pmid":"https://pubmed.ncbi.nlm.nih.gov/21300697"},"language":"en","primary_location":{"id":"doi:10.1093/bib/bbq073","is_oa":true,"landing_page_url":"https://doi.org/10.1093/bib/bbq073","pdf_url":"https://academic.oup.com/bib/article-pdf/12/3/189/715126/bbq073.pdf","source":{"id":"https://openalex.org/S91767247","display_name":"Briefings in Bioinformatics","issn_l":"1467-5463","issn":["1467-5463","1477-4054"],"is_oa":false,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Briefings in Bioinformatics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://academic.oup.com/bib/article-pdf/12/3/189/715126/bbq073.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113707070","display_name":"Peter J. Castaldi","orcid":null},"institutions":[{"id":"https://openalex.org/I1290900354","display_name":"Tufts Medical Center","ror":"https://ror.org/002hsbm82","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1290900354"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"P. J. Castaldi","raw_affiliation_strings":["Institute for Clinical Research and Health Policy Studies at Tufts Medical Center, USA"],"affiliations":[{"raw_affiliation_string":"Institute for Clinical Research and Health Policy Studies at Tufts Medical Center, USA","institution_ids":["https://openalex.org/I1290900354"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016796406","display_name":"Issa J Dahabreh","orcid":"https://orcid.org/0000-0002-2215-9931"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"I. J. Dahabreh","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5110876095","display_name":"J. P. A. Ioannidis","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"J. P. A. Ioannidis","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5113707070"],"corresponding_institution_ids":["https://openalex.org/I1290900354"],"apc_list":{"value":4011,"currency":"USD","value_usd":4011},"apc_paid":null,"fwci":4.5163,"has_fulltext":false,"cited_by_count":98,"citation_normalized_percentile":{"value":0.95664376,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"12","issue":"3","first_page":"189","last_page":"202"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10885","display_name":"Gene expression and cancer classification","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10885","display_name":"Gene expression and cancer classification","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10887","display_name":"Bioinformatics and Genomic Networks","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T13937","display_name":"Genetics, Bioinformatics, and Biomedical Research","score":0.9905999898910522,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/overfitting","display_name":"Overfitting","score":0.8617440462112427},{"id":"https://openalex.org/keywords/cross-validation","display_name":"Cross-validation","score":0.8543444871902466},{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"Generalizability theory","score":0.8424152135848999},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5964948534965515},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.5935287475585938},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.540002703666687},{"id":"https://openalex.org/keywords/logistic-regression","display_name":"Logistic regression","score":0.4561369717121124},{"id":"https://openalex.org/keywords/model-validation","display_name":"Model validation","score":0.42541688680648804},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.39583277702331543},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.3423570394515991},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.16750851273536682},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.07764002680778503}],"concepts":[{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.8617440462112427},{"id":"https://openalex.org/C27181475","wikidata":"https://www.wikidata.org/wiki/Q541014","display_name":"Cross-validation","level":2,"score":0.8543444871902466},{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.8424152135848999},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5964948534965515},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.5935287475585938},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.540002703666687},{"id":"https://openalex.org/C151956035","wikidata":"https://www.wikidata.org/wiki/Q1132755","display_name":"Logistic regression","level":2,"score":0.4561369717121124},{"id":"https://openalex.org/C3019813237","wikidata":"https://www.wikidata.org/wiki/Q65089264","display_name":"Model validation","level":2,"score":0.42541688680648804},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39583277702331543},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3423570394515991},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.16750851273536682},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.07764002680778503},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.0}],"mesh":[{"descriptor_ui":"D002965","descriptor_name":"Classification","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D002965","descriptor_name":"Classification","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D002965","descriptor_name":"Classification","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D002965","descriptor_name":"Classification","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D002965","descriptor_name":"Classification","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D002965","descriptor_name":"Classification","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D010363","descriptor_name":"Pattern Recognition, Automated","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D010363","descriptor_name":"Pattern Recognition, Automated","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D010363","descriptor_name":"Pattern Recognition, Automated","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D010363","descriptor_name":"Pattern Recognition, Automated","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D010363","descriptor_name":"Pattern Recognition, Automated","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D010363","descriptor_name":"Pattern Recognition, Automated","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D039781","descriptor_name":"PubMed","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D039781","descriptor_name":"PubMed","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D039781","descriptor_name":"PubMed","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D039781","descriptor_name":"PubMed","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D039781","descriptor_name":"PubMed","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D039781","descriptor_name":"PubMed","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D040901","descriptor_name":"Proteomics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D040901","descriptor_name":"Proteomics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D040901","descriptor_name":"Proteomics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D040901","descriptor_name":"Proteomics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D040901","descriptor_name":"Proteomics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D040901","descriptor_name":"Proteomics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":3,"locations":[{"id":"doi:10.1093/bib/bbq073","is_oa":true,"landing_page_url":"https://doi.org/10.1093/bib/bbq073","pdf_url":"https://academic.oup.com/bib/article-pdf/12/3/189/715126/bbq073.pdf","source":{"id":"https://openalex.org/S91767247","display_name":"Briefings in Bioinformatics","issn_l":"1467-5463","issn":["1467-5463","1477-4054"],"is_oa":false,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Briefings in Bioinformatics","raw_type":"journal-article"},{"id":"pmid:21300697","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/21300697","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Briefings in bioinformatics","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:3088312","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/3088312","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Brief Bioinform","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1093/bib/bbq073","is_oa":true,"landing_page_url":"https://doi.org/10.1093/bib/bbq073","pdf_url":"https://academic.oup.com/bib/article-pdf/12/3/189/715126/bbq073.pdf","source":{"id":"https://openalex.org/S91767247","display_name":"Briefings in Bioinformatics","issn_l":"1467-5463","issn":["1467-5463","1477-4054"],"is_oa":false,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Briefings in Bioinformatics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2108420460.pdf","grobid_xml":"https://content.openalex.org/works/W2108420460.grobid-xml"},"referenced_works_count":82,"referenced_works":["https://openalex.org/W1516312509","https://openalex.org/W1600041372","https://openalex.org/W1964703018","https://openalex.org/W1966525597","https://openalex.org/W1971348077","https://openalex.org/W1974618278","https://openalex.org/W1978097115","https://openalex.org/W1980337183","https://openalex.org/W1980649172","https://openalex.org/W1987024849","https://openalex.org/W1987485110","https://openalex.org/W1996647803","https://openalex.org/W1999555459","https://openalex.org/W2002401509","https://openalex.org/W2006761672","https://openalex.org/W2009157047","https://openalex.org/W2022406334","https://openalex.org/W2022441134","https://openalex.org/W2023595415","https://openalex.org/W2030165415","https://openalex.org/W2031175825","https://openalex.org/W2036756589","https://openalex.org/W2038407439","https://openalex.org/W2041700051","https://openalex.org/W2043235003","https://openalex.org/W2048040326","https://openalex.org/W2065267852","https://openalex.org/W2069709802","https://openalex.org/W2071009029","https://openalex.org/W2078769511","https://openalex.org/W2083602760","https://openalex.org/W2084764338","https://openalex.org/W2087231535","https://openalex.org/W2089821574","https://openalex.org/W2090084341","https://openalex.org/W2091959199","https://openalex.org/W2092342071","https://openalex.org/W2092926517","https://openalex.org/W2096891352","https://openalex.org/W2096910745","https://openalex.org/W2098533750","https://openalex.org/W2101687190","https://openalex.org/W2104092906","https://openalex.org/W2104451599","https://openalex.org/W2105017286","https://openalex.org/W2105981176","https://openalex.org/W2106160987","https://openalex.org/W2107328434","https://openalex.org/W2107956883","https://openalex.org/W2109034705","https://openalex.org/W2109103446","https://openalex.org/W2110298216","https://openalex.org/W2113629984","https://openalex.org/W2115452399","https://openalex.org/W2115944658","https://openalex.org/W2116302563","https://openalex.org/W2120766731","https://openalex.org/W2123034092","https://openalex.org/W2126872058","https://openalex.org/W2126930838","https://openalex.org/W2127952499","https://openalex.org/W2128985829","https://openalex.org/W2129152887","https://openalex.org/W2138482047","https://openalex.org/W2139336758","https://openalex.org/W2141007997","https://openalex.org/W2142495011","https://openalex.org/W2146791205","https://openalex.org/W2147625347","https://openalex.org/W2153043827","https://openalex.org/W2154195517","https://openalex.org/W2154899482","https://openalex.org/W2160891267","https://openalex.org/W2161371306","https://openalex.org/W2162713066","https://openalex.org/W2165117825","https://openalex.org/W2166038336","https://openalex.org/W2167098291","https://openalex.org/W2168914100","https://openalex.org/W2315123323","https://openalex.org/W2339926369","https://openalex.org/W4234673770"],"related_works":["https://openalex.org/W4220972140","https://openalex.org/W3161120485","https://openalex.org/W4223442957","https://openalex.org/W2149651625","https://openalex.org/W816105089","https://openalex.org/W2801469686","https://openalex.org/W2149903055","https://openalex.org/W2100523380","https://openalex.org/W1983416467","https://openalex.org/W4318240167"],"abstract_inverted_index":{"Proposed":[0],"molecular":[1,66,82,242],"classifiers":[2,83],"may":[3,39],"be":[4,43,48],"overfit":[5],"to":[6,19,111,120],"idiosyncrasies":[7],"of":[8,22,64,81,103,204,209,237,241],"noisy":[9],"genomic":[10],"and":[11,28,45,74,77,132,141,153,157,162,164,199,228],"proteomic":[12],"data.":[13],"Cross-validation":[14],"methods":[15,35],"are":[16,36,109,225],"often":[17],"used":[18],"obtain":[20],"estimates":[21],"classification":[23,149],"accuracy,":[24],"but":[25],"both":[26,94],"simulations":[27],"case":[29],"studies":[30,57,92,104,116,188,248],"suggest":[31],"that,":[32],"when":[33],"inappropriate":[34],"used,":[37],"bias":[38],"ensue.":[40],"Bias":[41],"can":[42,47],"bypassed":[44],"generalizability":[46],"tested":[49],"by":[50],"external":[51,62,88,133,239],"(independent)":[52],"validation.":[53,168,183],"We":[54,68,98],"evaluated":[55],"35":[56],"that":[58,100,108,217],"have":[59],"reported":[60,148],"on":[61,71],"validation":[63,89,134,208,240],"a":[65,122],"classifier.":[67],"extracted":[69],"information":[70],"study":[72,197],"design":[73],"methodological":[75],"features,":[76],"compared":[78],"the":[79,101,223],"performance":[80,150],"in":[84,125,160,195,222,231,245,250],"internal":[85,130],"cross-validation":[86,106,131,161,180,219],"versus":[87,181],"for":[90,151,166,179],"28":[91],"where":[93],"had":[95],"been":[96],"performed.":[97],"demonstrate":[99],"majority":[102],"pursued":[105],"practices":[107,220],"likely":[110],"overestimate":[112],"classifier":[113],"performance.":[114],"Most":[115],"were":[117],"markedly":[118],"underpowered":[119],"detect":[121],"20%":[123],"decrease":[124],"sensitivity":[126,152],"or":[127],"specificity":[128,154],"between":[129],"[median":[135],"power":[136],"was":[137,155,174],"36%":[138],"(IQR,":[139,143],"21-61%)":[140],"29%":[142],"15-65%),":[144],"respectively].":[145],"The":[146,169],"median":[147],"94%":[156],"98%,":[158],"respectively,":[159],"88%":[163],"81%":[165],"independent":[167,182,207],"relative":[170],"diagnostic":[171],"odds":[172],"ratio":[173],"3.26":[175],"(95%":[176],"CI":[177],"2.04-5.21)":[178],"Finally,":[184],"we":[185],"reviewed":[186],"all":[187],"(n":[189],"=":[190],"758)":[191],"which":[192],"cited":[193],"those":[194],"our":[196],"sample,":[198],"identified":[200],"only":[201],"one":[202],"instance":[203],"additional":[205],"subsequent":[206],"these":[210,214],"classifiers.":[211],"In":[212],"conclusion,":[213],"results":[215],"document":[216],"many":[218],"employed":[221],"literature":[224],"potentially":[226],"biased":[227],"genuine":[229],"progress":[230],"this":[232],"field":[233],"will":[234],"require":[235],"adoption":[236],"routine":[238],"classifiers,":[243],"preferably":[244],"much":[246],"larger":[247],"than":[249],"current":[251],"practice.":[252]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":6},{"year":2018,"cited_by_count":9},{"year":2017,"cited_by_count":6},{"year":2016,"cited_by_count":6},{"year":2015,"cited_by_count":3},{"year":2014,"cited_by_count":9},{"year":2013,"cited_by_count":8},{"year":2012,"cited_by_count":8}],"updated_date":"2026-03-10T16:38:18.471706","created_date":"2025-10-10T00:00:00"}