{"id":"https://openalex.org/W4411725760","doi":"https://doi.org/10.1109/jstsp.2025.3583140","title":"Measuring Training Variability From Stochastic Optimization Using Robust Nonparametric Testing","display_name":"Measuring Training Variability From Stochastic Optimization Using Robust Nonparametric Testing","publication_year":2025,"publication_date":"2025-06-27","ids":{"openalex":"https://openalex.org/W4411725760","doi":"https://doi.org/10.1109/jstsp.2025.3583140"},"language":"en","primary_location":{"id":"doi:10.1109/jstsp.2025.3583140","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2025.3583140","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049760570","display_name":"Sinjini Banerjee","orcid":"https://orcid.org/0000-0003-1028-4863"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sinjini Banerjee","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Rutgers, The State University of New Jersey, Piscataway, NJ, USA"],"raw_orcid":"https://orcid.org/0000-0003-1028-4863","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Rutgers, The State University of New Jersey, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024418285","display_name":"Tim Marrinan","orcid":"https://orcid.org/0000-0001-5240-3053"},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tim Marrinan","raw_affiliation_strings":["Pacific Northwest National Lab, Richland, WA, USA"],"raw_orcid":"https://orcid.org/0000-0001-5240-3053","affiliations":[{"raw_affiliation_string":"Pacific Northwest National Lab, Richland, WA, USA","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104327293","display_name":"Reilly Cannon","orcid":"https://orcid.org/0000-0002-8227-558X"},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Reilly Cannon","raw_affiliation_strings":["Pacific Northwest National Lab, Richland, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-8227-558X","affiliations":[{"raw_affiliation_string":"Pacific Northwest National Lab, Richland, WA, USA","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088679899","display_name":"Tony Chiang","orcid":"https://orcid.org/0000-0002-9817-2526"},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tony Chiang","raw_affiliation_strings":["Pacific Northwest National Laboratory and ARPA-H, Bethesda, MD, USA","Pacific Northwest National Laboratory and ARPA-H, Bethesda, Maryland, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory and ARPA-H, Bethesda, MD, USA","institution_ids":["https://openalex.org/I142606810"]},{"raw_affiliation_string":"Pacific Northwest National Laboratory and ARPA-H, Bethesda, Maryland, USA","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087162046","display_name":"Anand D. Sarwate","orcid":"https://orcid.org/0000-0001-6123-5282"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anand D. Sarwate","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Rutgers, The State University of New Jersey, Piscataway, NJ, USA"],"raw_orcid":"https://orcid.org/0000-0001-6123-5282","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Rutgers, The State University of New Jersey, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5049760570"],"corresponding_institution_ids":["https://openalex.org/I102322142"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.14071746,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"19","issue":"7","first_page":"1477","last_page":"1492"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11890","display_name":"Scientific Measurement and Uncertainty Evaluation","score":0.9294999837875366,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11890","display_name":"Scientific Measurement and Uncertainty Evaluation","score":0.9294999837875366,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/nonparametric-statistics","display_name":"Nonparametric statistics","score":0.6942512392997742},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.606439471244812},{"id":"https://openalex.org/keywords/stochastic-optimization","display_name":"Stochastic optimization","score":0.5017831325531006},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5016391277313232},{"id":"https://openalex.org/keywords/robust-optimization","display_name":"Robust optimization","score":0.44450095295906067},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4171200692653656},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3720465898513794},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.3248979449272156},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.312728613615036},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.27554357051849365}],"concepts":[{"id":"https://openalex.org/C102366305","wikidata":"https://www.wikidata.org/wiki/Q1097688","display_name":"Nonparametric statistics","level":2,"score":0.6942512392997742},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.606439471244812},{"id":"https://openalex.org/C194387892","wikidata":"https://www.wikidata.org/wiki/Q1747770","display_name":"Stochastic optimization","level":2,"score":0.5017831325531006},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5016391277313232},{"id":"https://openalex.org/C193254401","wikidata":"https://www.wikidata.org/wiki/Q2160088","display_name":"Robust optimization","level":2,"score":0.44450095295906067},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4171200692653656},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3720465898513794},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3248979449272156},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.312728613615036},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.27554357051849365}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/jstsp.2025.3583140","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2025.3583140","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4271113244","display_name":null,"funder_award_id":"DE-AC05-76RL0-1830","funder_id":"https://openalex.org/F4320306250","funder_display_name":"Battelle"}],"funders":[{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320306250","display_name":"Battelle","ror":"https://ror.org/01h5tnr73"},{"id":"https://openalex.org/F4320337547","display_name":"Laboratory Directed Research and Development","ror":"https://ror.org/01e41cf67"},{"id":"https://openalex.org/F4320338354","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W1987452562","https://openalex.org/W1988753456","https://openalex.org/W1990216275","https://openalex.org/W1992061133","https://openalex.org/W2039446404","https://openalex.org/W2098824882","https://openalex.org/W2124715517","https://openalex.org/W2254249950","https://openalex.org/W2754517384","https://openalex.org/W2923972789","https://openalex.org/W2985190846","https://openalex.org/W3138516171","https://openalex.org/W4213251304","https://openalex.org/W4285579170"],"related_works":["https://openalex.org/W3005017983","https://openalex.org/W2205708137","https://openalex.org/W3182145356","https://openalex.org/W3212504317","https://openalex.org/W1988872022","https://openalex.org/W2224558845","https://openalex.org/W2140496876","https://openalex.org/W3017736765","https://openalex.org/W193468460","https://openalex.org/W3028487898"],"abstract_inverted_index":{"Deep":[0],"neural":[1],"network":[2],"training":[3,22,137],"often":[4],"involves":[5],"stochastic":[6],"optimization,":[7],"meaning":[8],"each":[9],"run":[10],"will":[11],"produce":[12],"a":[13,33,59,65,114,148],"different":[14],"model.":[15],"This":[16],"implies":[17],"that":[18,144,172],"hyperparameters":[19],"of":[20,113,116,151,193,204],"the":[21,26,37,40,69,81,91,94,110,130,152,162,202],"process,":[23],"such":[24,49],"as":[25,50,124],"random":[27,196],"seed":[28,197],"itself,":[29],"can":[30,53],"potentially":[31],"have":[32],"significant":[34],"influence":[35],"on":[36],"variability":[38,168],"in":[39,198],"trained":[41,118],"models.":[42],"Measuring":[43],"model":[44,74,154,167],"quality":[45],"by":[46,102],"summary":[47,67],"statistics,":[48],"test":[51],"accuracy,":[52,182],"obscure":[54],"this":[55,100,122],"dependence.":[56],"We":[57,127,156],"propose":[58],"robust":[60],"hypothesis":[61,77],"testing":[62,78],"framework":[63,98],"and":[64,120,169],"novel":[66],"statistic,":[68],"\u03b1-trimming":[70,82,131,163],"level,":[71],"to":[72,133,142,160,165],"measure":[73,166],"similarity.":[75],"Applying":[76],"directly":[79],"with":[80],"level":[83,132,164],"is":[84,147,174],"challenging":[85],"because":[86],"we":[87],"cannot":[88],"accurately":[89],"describe":[90],"distribution":[92,108,112],"under":[93],"null":[95],"hypothesis.":[96],"Our":[97],"addresses":[99],"issue":[101],"determining":[103],"how":[104,135,159],"closely":[105],"an":[106,145],"approximate":[107],"resembles":[109],"expected":[111,185],"group":[115],"individually":[117],"models":[119],"using":[121],"approximation":[123],"our":[125,205],"reference.":[126],"then":[128],"use":[129,161],"suggest":[134],"many":[136],"runs":[138],"should":[139],"be":[140],"sampled":[141],"ensure":[143],"ensemble":[146],"reliable":[149],"representative":[150],"true":[153],"performance.":[155],"also":[157],"show":[158],"demonstrate":[170],"experimentally":[171],"it":[173],"more":[175],"expressive":[176],"than":[177],"performance":[178],"metrics":[179],"like":[180],"validation":[181],"churn,":[183],"or":[184],"calibration":[186],"error":[187],"when":[188],"taken":[189],"alone.":[190],"An":[191],"application":[192],"fine-tuning":[194],"over":[195],"transfer":[199],"learning":[200],"illustrates":[201],"advantage":[203],"new":[206],"metric.":[207]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}