{"id":"https://openalex.org/W4403579871","doi":"https://doi.org/10.48550/arxiv.2410.13609","title":"All models are wrong, some are useful: Model Selection with Limited Labels","display_name":"All models are wrong, some are useful: Model Selection with Limited Labels","publication_year":2024,"publication_date":"2024-10-17","ids":{"openalex":"https://openalex.org/W4403579871","doi":"https://doi.org/10.48550/arxiv.2410.13609"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.13609","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.13609","pdf_url":"https://arxiv.org/pdf/2410.13609","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.13609","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007711261","display_name":"Patrik Okanovic","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Okanovic, Patrik","raw_affiliation_strings":["ETH Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Kirsch, Andreas","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kirsch, Andreas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Kasper, Jannes","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Kasper, Jannes","raw_affiliation_strings":["TU Delft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TU Delft","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026990786","display_name":"Torsten Hoefler","orcid":"https://orcid.org/0000-0002-1333-9797","observed_orcids":["https://orcid.org/0000-0002-1333-9797","https://orcid.org/0000-0001-9611-7171"]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Hoefler, Torsten","raw_affiliation_strings":["ETH Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003040843","display_name":"Andreas Krause","orcid":"https://orcid.org/0000-0001-7260-9673","observed_orcids":["https://orcid.org/0000-0001-7260-9673"]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Krause, Andreas","raw_affiliation_strings":["ETH Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016955253","display_name":"Nezihe Merve G\u00fcrel","orcid":"https://orcid.org/0000-0002-4747-2406","observed_orcids":["https://orcid.org/0000-0002-4747-2406"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"G\u00fcrel, Nezihe Merve","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.5365999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.5365999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.365200012922287,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.09200000017881393,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/model-selection","display_name":"model selection","score":0.8999999761581421},{"id":"https://openalex.org/keywords/label-efficient-learning","display_name":"label-efficient learning","score":0.6150000095367432},{"id":"https://openalex.org/keywords/active-learning-machine-learning","display_name":"active learning (machine learning)","score":0.75},{"id":"https://openalex.org/keywords/unlabeled-target-data","display_name":"unlabeled target data","score":0.7760000228881836},{"id":"https://openalex.org/keywords/transfer-learning","display_name":"transfer learning","score":0.6230000257492065}],"concepts":[{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.6742002964019775},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.5979715585708618},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5302820801734924},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36603742837905884}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.13609","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.13609","pdf_url":"https://arxiv.org/pdf/2410.13609","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.13609","doi":"https://doi.org/10.48550/arxiv.2410.13609","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.13609","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.13609","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.13609","pdf_url":"https://arxiv.org/pdf/2410.13609","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320331518","display_name":"Centro Svizzero di Calcolo Scientifico","ror":"https://ror.org/04rzmms09"},{"id":"https://openalex.org/F4320331617","display_name":"National Supercomputing Centre Singapore","ror":"https://ror.org/05qbsk739"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403579871.pdf","grobid_xml":"https://content.openalex.org/works/W4403579871.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W1872312298","https://openalex.org/W1995812641","https://openalex.org/W2549139847","https://openalex.org/W2612445135","https://openalex.org/W2799054028","https://openalex.org/W2896457183","https://openalex.org/W2946948417","https://openalex.org/W2951061410","https://openalex.org/W2951454049","https://openalex.org/W2964317943","https://openalex.org/W2965373594","https://openalex.org/W2978017171","https://openalex.org/W3030163527","https://openalex.org/W3032062202","https://openalex.org/W3135367836","https://openalex.org/W4226014440","https://openalex.org/W4321012582","https://openalex.org/W4387995065","https://openalex.org/W4390784180","https://openalex.org/W4391871787","https://openalex.org/W4407814500","https://openalex.org/W4415795364","https://openalex.org/W4415796722","https://openalex.org/W6945222046","https://openalex.org/W7103748778"],"related_works":["https://openalex.org/W4404608593","https://openalex.org/W7103748778","https://openalex.org/W4389363924","https://openalex.org/W7095148356","https://openalex.org/W4414922313","https://openalex.org/W4385428276","https://openalex.org/W3094252010","https://openalex.org/W7057732900","https://openalex.org/W7095350012","https://openalex.org/W3126679997"],"abstract_inverted_index":{"We":[0],"introduce":[1],"MODEL":[2,19,52,83,113],"SELECTOR,":[3],"a":[4,13,22,130],"framework":[5],"for":[6,29,40,58],"label-efficient":[7],"selection":[8],"of":[9,15,25,102,112,139],"pretrained":[10,38,81],"classifiers.":[11],"Given":[12],"pool":[14],"unlabeled":[16],"target":[17,44],"data,":[18],"SELECTOR":[20,53,84,114],"samples":[21],"small":[23],"subset":[24],"highly":[26],"informative":[27],"examples":[28],"labeling,":[30],"in":[31,115],"order":[32],"to":[33,91,93,99,126],"efficiently":[34],"identify":[35,94],"the":[36,56,64,86,95,100,103,110,121,140],"best":[37,65,96,141],"model":[39,72,97,116],"deployment":[41],"on":[42,74],"this":[43],"dataset.":[45],"Through":[46],"extensive":[47],"experiments,":[48],"we":[49],"demonstrate":[50],"that":[51],"drastically":[54],"reduces":[55,85,120],"need":[57],"labeled":[59],"data":[60],"while":[61],"consistently":[62],"picking":[63],"or":[66],"near-best":[67,131],"performing":[68],"model.":[69,142],"Across":[70],"18":[71],"collections":[73],"16":[75],"different":[76],"datasets,":[77],"comprising":[78],"over":[79],"1,500":[80],"models,":[82],"labeling":[87,122],"cost":[88,101,123],"by":[89,124],"up":[90,125],"94.15%":[92],"compared":[98],"strongest":[104],"baseline.":[105],"Our":[106],"results":[107],"further":[108],"highlight":[109],"robustness":[111],"selection,":[117],"as":[118],"it":[119],"72.41%":[127],"when":[128],"selecting":[129],"model,":[132],"whose":[133],"accuracy":[134],"is":[135],"only":[136],"within":[137],"1%":[138]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-10-21T00:00:00"}