{"id":"https://openalex.org/W2791137381","doi":"https://doi.org/10.1117/12.2293818","title":"Test data reuse for evaluation of adaptive machine learning algorithms: over-fitting to a fixed 'test' dataset and a potential solution","display_name":"Test data reuse for evaluation of adaptive machine learning algorithms: over-fitting to a fixed 'test' dataset and a potential solution","publication_year":2018,"publication_date":"2018-03-07","ids":{"openalex":"https://openalex.org/W2791137381","doi":"https://doi.org/10.1117/12.2293818","mag":"2791137381"},"language":"en","primary_location":{"id":"doi:10.1117/12.2293818","is_oa":false,"landing_page_url":"https://doi.org/10.1117/12.2293818","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Medical Imaging 2018: Image Perception, Observer Performance, and Technology Assessment","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091213912","display_name":"Alexej Gossmann","orcid":"https://orcid.org/0000-0001-9068-3877"},"institutions":[{"id":"https://openalex.org/I114832834","display_name":"Tulane University","ror":"https://ror.org/04vmvtb21","country_code":"US","type":"education","lineage":["https://openalex.org/I114832834"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Alexej Gossmann","raw_affiliation_strings":["Tulane Univ. (United States)"],"affiliations":[{"raw_affiliation_string":"Tulane Univ. (United States)","institution_ids":["https://openalex.org/I114832834"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074995495","display_name":"Aria Pezeshk","orcid":"https://orcid.org/0000-0002-3570-3051"},"institutions":[{"id":"https://openalex.org/I1320320070","display_name":"United States Food and Drug Administration","ror":"https://ror.org/034xvzb47","country_code":"US","type":"government","lineage":["https://openalex.org/I1299022934","https://openalex.org/I1320320070"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aria Pezeshk","raw_affiliation_strings":["U.S. Food and Drug Administration (United States)"],"affiliations":[{"raw_affiliation_string":"U.S. Food and Drug Administration (United States)","institution_ids":["https://openalex.org/I1320320070"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5073468417","display_name":"Berkman Sahiner","orcid":"https://orcid.org/0000-0003-2804-2264"},"institutions":[{"id":"https://openalex.org/I1320320070","display_name":"United States Food and Drug Administration","ror":"https://ror.org/034xvzb47","country_code":"US","type":"government","lineage":["https://openalex.org/I1299022934","https://openalex.org/I1320320070"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Berkman Sahiner","raw_affiliation_strings":["U.S. Food and Drug Administration (United States)"],"affiliations":[{"raw_affiliation_string":"U.S. Food and Drug Administration (United States)","institution_ids":["https://openalex.org/I1320320070"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5091213912"],"corresponding_institution_ids":["https://openalex.org/I114832834"],"apc_list":null,"apc_paid":null,"fwci":1.6288,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.87324873,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"19","last_page":"19"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.9930999875068665,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10243","display_name":"Statistical Methods and Bayesian Inference","score":0.9878000020980835,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/overfitting","display_name":"Overfitting","score":0.9641284346580505},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7452222108840942},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.667658269405365},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6495807766914368},{"id":"https://openalex.org/keywords/test-data","display_name":"Test data","score":0.6354471445083618},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.6335887312889099},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.5579504370689392},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5575222969055176},{"id":"https://openalex.org/keywords/reuse","display_name":"Reuse","score":0.5395945310592651},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5366655588150024},{"id":"https://openalex.org/keywords/population","display_name":"Population","score":0.5335695147514343},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.36411046981811523},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.17233598232269287},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14790686964988708},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07099923491477966}],"concepts":[{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.9641284346580505},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7452222108840942},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.667658269405365},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6495807766914368},{"id":"https://openalex.org/C16910744","wikidata":"https://www.wikidata.org/wiki/Q7705759","display_name":"Test data","level":2,"score":0.6354471445083618},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.6335887312889099},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5579504370689392},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5575222969055176},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.5395945310592651},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5366655588150024},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.5335695147514343},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36411046981811523},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.17233598232269287},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14790686964988708},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07099923491477966},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C548081761","wikidata":"https://www.wikidata.org/wiki/Q180388","display_name":"Waste management","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1117/12.2293818","is_oa":false,"landing_page_url":"https://doi.org/10.1117/12.2293818","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Medical Imaging 2018: Image Perception, Observer Performance, and Technology Assessment","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W158824538","https://openalex.org/W429766147","https://openalex.org/W1480487340","https://openalex.org/W1555759181","https://openalex.org/W1663583528","https://openalex.org/W1798749056","https://openalex.org/W1982723861","https://openalex.org/W2015056255","https://openalex.org/W2015866962","https://openalex.org/W2046875666","https://openalex.org/W2110065044","https://openalex.org/W2112076978","https://openalex.org/W2122441470","https://openalex.org/W2134305330","https://openalex.org/W2144981148","https://openalex.org/W2154776925","https://openalex.org/W2155653793","https://openalex.org/W2157825442","https://openalex.org/W2160453617","https://openalex.org/W2225981128","https://openalex.org/W2258658829","https://openalex.org/W2322006099","https://openalex.org/W2527726942","https://openalex.org/W2550828940","https://openalex.org/W2911964244","https://openalex.org/W2951081425","https://openalex.org/W2963389298","https://openalex.org/W2964326026","https://openalex.org/W3014064419","https://openalex.org/W4242521992","https://openalex.org/W4252278281","https://openalex.org/W4294541781","https://openalex.org/W4295836186","https://openalex.org/W4298090737","https://openalex.org/W4298304654","https://openalex.org/W4399271987","https://openalex.org/W6606574684","https://openalex.org/W6628633436","https://openalex.org/W6636906085","https://openalex.org/W6638392197","https://openalex.org/W6646147174","https://openalex.org/W6676591658","https://openalex.org/W6676769703","https://openalex.org/W6679825467","https://openalex.org/W6679986405","https://openalex.org/W6682981795","https://openalex.org/W6684759661","https://openalex.org/W6688875923","https://openalex.org/W6692563690","https://openalex.org/W6695740115","https://openalex.org/W6738800795","https://openalex.org/W6739031823","https://openalex.org/W6870084398"],"related_works":["https://openalex.org/W4362597605","https://openalex.org/W1574414179","https://openalex.org/W4297676672","https://openalex.org/W3009056573","https://openalex.org/W2922073769","https://openalex.org/W4281702477","https://openalex.org/W2167608453","https://openalex.org/W2075445622","https://openalex.org/W2791137381","https://openalex.org/W2973011565"],"abstract_inverted_index":{"After":[0],"the":[1,9,18,27,47,61,128,131,145,162,167,170,188,199,203,214,224,228,242,247,251,255,263,272,276,279,283],"initial":[2],"release":[3],"of":[4,39,85,169,173,187,198,223,244,257,271],"a":[5,36,75,99,104,116,137,153,195,258],"machine":[6],"learning":[7],"algorithm,":[8],"subsequently":[10],"gathered":[11],"data":[12,72,121,249],"can":[13,123,217],"be":[14,65,218],"used":[15],"to":[16,23,35,58,64,91,97,119,130,152,179,220,227,246,275],"augment":[17],"training":[19],"dataset":[20,102,216],"in":[21,126,155],"order":[22],"modify":[24],"or":[25],"fine-tune":[26],"algorithm.":[28],"For":[29],"algorithm":[30,62,129,163],"performance":[31,77,106,139,200,285],"evaluation":[32,78],"that":[33,54,237],"generalizes":[34],"targeted":[37,48],"population":[38,49],"cases,":[40],"ideally,":[41],"test":[42,55,71,83,101,120,132,146,190,204,215,248,265],"datasets":[43,84],"randomly":[44],"drawn":[45],"from":[46,144],"are":[50,88],"used.":[51],"To":[52],"ensure":[53,221],"results":[56],"generalize":[57],"new":[59,68,76,105],"data,":[60,133,205],"needs":[63],"evaluated":[66],"on":[67,202,210,262],"and":[69,93,157,206],"independent":[70],"each":[73],"time":[74],"is":[79,95,141],"required.":[80],"However,":[81],"medical":[82],"sufficient":[86],"quality":[87],"often":[89],"hard":[90],"acquire,":[92],"it":[94],"tempting":[96],"utilize":[98],"previously-used":[100],"for":[103],"evaluation.":[107],"With":[108,231],"extensive":[109,232],"simulation":[110,233,252],"studies,":[111,234],"we":[112,235],"illustrate":[113],"how":[114,211],"such":[115],"\"naive\"":[117],"approach":[118],"reuse":[122,186],"inadvertently":[124],"result":[125],"overfitting":[127,149,245],"even":[134],"when":[135],"only":[136],"global":[138],"metric":[140,201],"reported":[142,225,264,284],"back":[143],"dataset.":[147,191],"The":[148],"behavior":[150],"leads":[151],"loss":[154],"generalization":[156,222],"overly":[158],"optimistic":[159],"conclusions":[160],"about":[161],"performance.":[164,266],"We":[165,267],"investigate":[166],"use":[168],"Thresholdout":[171,183,238],"method":[172],"Dwork":[174],"et.":[175],"al.":[176],"(Ref.":[177],"1)":[178],"tackle":[180],"this":[181],"problem.":[182],"allows":[184],"repeated":[185],"same":[189],"It":[192],"essentially":[193],"reports":[194],"noisy":[196],"version":[197],"provides":[207],"theoretical":[208,273],"guarantees":[209,274],"many":[212],"times":[213],"accessed":[219],"answers":[226],"underlying":[229],"distribution.":[230],"show":[236],"indeed":[239],"substantially":[240],"reduces":[241],"problem":[243],"under":[250,278],"conditions,":[253],"at":[254],"cost":[256],"mild":[259],"additional":[260],"uncertainty":[261],"also":[268],"extend":[269],"some":[270],"area":[277],"ROC":[280],"curve":[281],"as":[282],"metric.":[286]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":6},{"year":2018,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}