{"id":"https://openalex.org/W4413826203","doi":"https://doi.org/10.14778/3725688.3725689","title":"A Systematic Study on Early Stopping Metrics in HPO and the Implications of Uncertainty","display_name":"A Systematic Study on Early Stopping Metrics in HPO and the Implications of Uncertainty","publication_year":2025,"publication_date":"2025-02-01","ids":{"openalex":"https://openalex.org/W4413826203","doi":"https://doi.org/10.14778/3725688.3725689"},"language":"en","primary_location":{"id":"doi:10.14778/3725688.3725689","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3725688.3725689","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037842747","display_name":"Jiawei Guan","orcid":"https://orcid.org/0000-0002-7538-9722"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Jiawei Guan","raw_affiliation_strings":["Renmin University of China"],"affiliations":[{"raw_affiliation_string":"Renmin University of China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100401308","display_name":"Feng Zhang","orcid":"https://orcid.org/0000-0003-1475-8480"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Feng Zhang","raw_affiliation_strings":["Renmin University of China"],"affiliations":[{"raw_affiliation_string":"Renmin University of China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047469209","display_name":"Jiesong Liu","orcid":"https://orcid.org/0000-0002-8311-020X"},"institutions":[{"id":"https://openalex.org/I137902535","display_name":"North Carolina State University","ror":"https://ror.org/04tj63d06","country_code":"US","type":"education","lineage":["https://openalex.org/I137902535"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiesong Liu","raw_affiliation_strings":["North Carolina State University"],"affiliations":[{"raw_affiliation_string":"North Carolina State University","institution_ids":["https://openalex.org/I137902535"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008721449","display_name":"Xiaoyong Du","orcid":"https://orcid.org/0000-0002-5757-9135"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoyong Du","raw_affiliation_strings":["Renmin University of China"],"affiliations":[{"raw_affiliation_string":"Renmin University of China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100624451","display_name":"Xipeng Shen","orcid":"https://orcid.org/0000-0003-3599-8010"},"institutions":[{"id":"https://openalex.org/I137902535","display_name":"North Carolina State University","ror":"https://ror.org/04tj63d06","country_code":"US","type":"education","lineage":["https://openalex.org/I137902535"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xipeng Shen","raw_affiliation_strings":["North Carolina State University"],"affiliations":[{"raw_affiliation_string":"North Carolina State University","institution_ids":["https://openalex.org/I137902535"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5037842747"],"corresponding_institution_ids":["https://openalex.org/I78988378"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.13197115,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"18","issue":"6","first_page":"1551","last_page":"1564"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.9793999791145325,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12205","display_name":"Time Series Analysis and Forecasting","score":0.974399983882904,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.38873428106307983},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.36739927530288696},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.3268008232116699},{"id":"https://openalex.org/keywords/environmental-science","display_name":"Environmental science","score":0.321344792842865},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3212848901748657}],"concepts":[{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.38873428106307983},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.36739927530288696},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3268008232116699},{"id":"https://openalex.org/C39432304","wikidata":"https://www.wikidata.org/wiki/Q188847","display_name":"Environmental science","level":0,"score":0.321344792842865},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3212848901748657}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.14778/3725688.3725689","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3725688.3725689","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W1536741282","https://openalex.org/W1881549354","https://openalex.org/W2001474264","https://openalex.org/W2103012681","https://openalex.org/W2769041395","https://openalex.org/W2903513346","https://openalex.org/W2909693411","https://openalex.org/W2949676527","https://openalex.org/W2959716049","https://openalex.org/W2967663220","https://openalex.org/W2970851599","https://openalex.org/W3014596384","https://openalex.org/W3036253705","https://openalex.org/W3045004532","https://openalex.org/W3102100346","https://openalex.org/W3115207133","https://openalex.org/W3134103293","https://openalex.org/W3134774296","https://openalex.org/W3137081859","https://openalex.org/W3139250374","https://openalex.org/W3155731460","https://openalex.org/W3164731060","https://openalex.org/W3175777295","https://openalex.org/W3179950556","https://openalex.org/W3210146484","https://openalex.org/W3213763152","https://openalex.org/W4221062824","https://openalex.org/W4221146709","https://openalex.org/W4281753793","https://openalex.org/W4283322925","https://openalex.org/W4285451014","https://openalex.org/W4286447321","https://openalex.org/W4288104008","https://openalex.org/W4315628777","https://openalex.org/W4386768622","https://openalex.org/W4389609792","https://openalex.org/W4398234519","https://openalex.org/W4400015470","https://openalex.org/W4400925671","https://openalex.org/W4404239715","https://openalex.org/W4411374627"],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4391375266","https://openalex.org/W1979597421","https://openalex.org/W2007980826","https://openalex.org/W2061531152","https://openalex.org/W3002753104","https://openalex.org/W2077600819","https://openalex.org/W2142036596","https://openalex.org/W2072657027"],"abstract_inverted_index":{"The":[0],"development":[1],"of":[2,44,51,68,71,77,86,100,152,177,187,208,217],"hyperparameter":[3],"optimization":[4],"(HPO)":[5],"algorithms":[6],"is":[7],"an":[8,118,149],"important":[9],"topic":[10],"within":[11],"both":[12],"the":[13,42,49,58,69,75,98,123,136,146,174,184,204],"machine":[14],"learning":[15],"and":[16,91,109,164,206],"data":[17],"management":[18],"domains.":[19],"While":[20],"numerous":[21],"strategies":[22],"employing":[23],"early":[24,45,52,78,101,119,124,220],"stopping":[25,46,53,102,120],"mechanisms":[26,218],"have":[27],"been":[28],"proposed":[29],"to":[30,132,135,154,212],"bolster":[31],"HPO":[32,60,108,128,169],"efficiency,":[33],"there":[34],"remains":[35],"a":[36,65,84,191,201,213],"notable":[37],"deficiency":[38],"in":[39,96,122,167],"understanding":[40],"how":[41],"selection":[43,73,179,205],"metrics":[47,87],"influences":[48],"reliability":[50,99],"decisions":[54],"and,":[55],"by":[56,130],"extension,":[57],"broader":[59],"outcomes.":[61],"This":[62,193],"paper":[63],"undertakes":[64],"systematic":[66],"exploration":[67],"impact":[70],"metric":[72,121,147,178],"on":[74,107,183],"effectiveness":[76],"stopping-based":[79,221],"HPO.":[80,222],"Specifically,":[81],"we":[82],"introduce":[83],"set":[85],"that":[88,113,198],"incorporate":[89],"uncertainty":[90,144,189],"highlight":[92],"their":[93],"practical":[94],"significance":[95],"enhancing":[97],"decisions.":[103],"Our":[104],"empirical":[105,196],"experiments":[106],"NAS":[110],"benchmarks":[111],"show":[112],"using":[114],"training":[115,125],"loss":[116],"as":[117,190,200],"stages":[126],"improves":[127],"outcomes":[129],"up":[131,153],"24.76%":[133],"compared":[134],"more":[137,214],"widely":[138],"accepted":[139],"validation":[140],"loss.":[141],"Furthermore,":[142],"integrating":[143,188],"into":[145,160],"yields":[148],"additional":[150],"improvement":[151],"4%":[155],"under":[156],"budget":[157],"constraints,":[158],"translating":[159],"meaningful":[161],"resource":[162],"savings":[163],"scalability":[165],"benefits":[166],"large-scale":[168],"scenarios.":[170],"These":[171],"findings":[172],"demonstrate":[173],"critical":[175],"role":[176],"while":[180],"shedding":[181],"light":[182],"potential":[185],"implications":[186],"metric.":[192],"research":[194],"provides":[195],"insights":[197],"serve":[199],"compass":[202],"for":[203],"formulation":[207],"metrics,":[209],"thereby":[210],"contributing":[211],"profound":[215],"comprehension":[216],"underpinning":[219]},"counts_by_year":[],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}