{"id":"https://openalex.org/W4387995335","doi":"https://doi.org/10.48550/arxiv.2310.17653","title":"Fantastic Gains and Where to Find Them: On the Existence and Prospect of General Knowledge Transfer between Any Pretrained Model","display_name":"Fantastic Gains and Where to Find Them: On the Existence and Prospect of General Knowledge Transfer between Any Pretrained Model","publication_year":2023,"publication_date":"2023-10-26","ids":{"openalex":"https://openalex.org/W4387995335","doi":"https://doi.org/10.48550/arxiv.2310.17653"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2310.17653","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.17653","pdf_url":"https://arxiv.org/pdf/2310.17653","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2310.17653","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103125177","display_name":"Karsten Roth","orcid":"https://orcid.org/0000-0003-1510-7217","observed_orcids":["https://orcid.org/0000-0003-1510-7217"]},"institutions":[{"id":"https://openalex.org/I4401726832","display_name":"T\u00fcbingen AI Center","ror":"https://ror.org/0107nyd78","country_code":"DE","type":"education","lineage":["https://openalex.org/I149899117","https://openalex.org/I4210135521","https://openalex.org/I4401726832","https://openalex.org/I8087733"]},{"id":"https://openalex.org/I8087733","display_name":"University of T\u00fcbingen","ror":"https://ror.org/03a1kwz48","country_code":"DE","type":"education","lineage":["https://openalex.org/I8087733"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Roth, Karsten","raw_affiliation_strings":["T\u00fcbingen AI Center & University of T\u00fcbingen ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"T\u00fcbingen AI Center & University of T\u00fcbingen ,","institution_ids":["https://openalex.org/I4401726832","https://openalex.org/I8087733"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039022463","display_name":"Lukas Thede","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4401726832","display_name":"T\u00fcbingen AI Center","ror":"https://ror.org/0107nyd78","country_code":"DE","type":"education","lineage":["https://openalex.org/I149899117","https://openalex.org/I4210135521","https://openalex.org/I4401726832","https://openalex.org/I8087733"]},{"id":"https://openalex.org/I8087733","display_name":"University of T\u00fcbingen","ror":"https://ror.org/03a1kwz48","country_code":"DE","type":"education","lineage":["https://openalex.org/I8087733"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Thede, Lukas","raw_affiliation_strings":["T\u00fcbingen AI Center & University of T\u00fcbingen ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"T\u00fcbingen AI Center & University of T\u00fcbingen ,","institution_ids":["https://openalex.org/I4401726832","https://openalex.org/I8087733"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001366802","display_name":"A. Sophia Koepke","orcid":"https://orcid.org/0000-0002-5807-0576","observed_orcids":["https://orcid.org/0000-0002-5807-0576"]},"institutions":[{"id":"https://openalex.org/I4401726832","display_name":"T\u00fcbingen AI Center","ror":"https://ror.org/0107nyd78","country_code":"DE","type":"education","lineage":["https://openalex.org/I149899117","https://openalex.org/I4210135521","https://openalex.org/I4401726832","https://openalex.org/I8087733"]},{"id":"https://openalex.org/I8087733","display_name":"University of T\u00fcbingen","ror":"https://ror.org/03a1kwz48","country_code":"DE","type":"education","lineage":["https://openalex.org/I8087733"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Koepke, Almut Sophia","raw_affiliation_strings":["T\u00fcbingen AI Center & University of T\u00fcbingen ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"T\u00fcbingen AI Center & University of T\u00fcbingen ,","institution_ids":["https://openalex.org/I4401726832","https://openalex.org/I8087733"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003562101","display_name":"Oriol Vinyals","orcid":"https://orcid.org/0000-0001-7848-7283","observed_orcids":["https://orcid.org/0000-0001-7848-7283"]},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Vinyals, Oriol","raw_affiliation_strings":["Google DeepMind"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google DeepMind","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081732358","display_name":"Olivier J. H\u00e9naff","orcid":"https://orcid.org/0000-0001-8183-9489","observed_orcids":["https://orcid.org/0000-0001-8183-9489"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"H\u00e9naff, Olivier","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5040372929","display_name":"Zeynep Akata","orcid":"https://orcid.org/0000-0002-1432-7747","observed_orcids":["https://orcid.org/0000-0002-1432-7747"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Akata, Zeynep","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.885699987411499,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.885699987411499,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.10639999806880951,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.005200000014156103,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/knowledge-transfer","display_name":"knowledge transfer","score":0.6389999985694885},{"id":"https://openalex.org/keywords/pre-trained-models","display_name":"pre-trained models","score":0.847000002861023},{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":0.7649999856948853},{"id":"https://openalex.org/keywords/data-partitioning","display_name":"data partitioning","score":0.7009999752044678},{"id":"https://openalex.org/keywords/feature-sets","display_name":"feature sets","score":0.4560000002384186},{"id":"https://openalex.org/keywords/knowledge-fusion","display_name":"knowledge fusion","score":0.6010000109672546},{"id":"https://openalex.org/keywords/model-library","display_name":"model library","score":0.42500001192092896},{"id":"https://openalex.org/keywords/transfer-learning","display_name":"transfer learning","score":0.3109999895095825}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8070720434188843},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.678444504737854},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6292743682861328},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6185070276260376},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.5820878744125366},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5669504404067993},{"id":"https://openalex.org/C2776960227","wikidata":"https://www.wikidata.org/wiki/Q2586354","display_name":"Knowledge transfer","level":2,"score":0.5036737322807312},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.47560247778892517},{"id":"https://openalex.org/C45493050","wikidata":"https://www.wikidata.org/wiki/Q7884934","display_name":"Unified Model","level":2,"score":0.4181561768054962},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.07426825165748596},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2310.17653","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.17653","pdf_url":"https://arxiv.org/pdf/2310.17653","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.17653","doi":"https://doi.org/10.48550/arxiv.2310.17653","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.17653","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2310.17653","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.17653","pdf_url":"https://arxiv.org/pdf/2310.17653","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1562512020","display_name":null,"funder_award_id":"206/4-1","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G4106808123","display_name":"Deeply Explainable Intelligent Machines","funder_award_id":"853489","funder_id":"https://openalex.org/F4320338335","funder_display_name":"H2020 European Research Council"},{"id":"https://openalex.org/G53513784","display_name":"SFB 1233: Robustheit des Sehens  Prinzipien der Inferenz und neuronale Mechanismen","funder_award_id":"276693517","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G5415233182","display_name":"Verbundprojekt TUEAI: T\u00fcbingen AI Center","funder_award_id":"01IS18039A","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"},{"id":"https://openalex.org/G7907426250","display_name":"EXC 2064: Maschinelles Lernen: Neue Perspektiven f\u00fcr die Wissenschaft","funder_award_id":"390727645","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G864726782","display_name":null,"funder_award_id":"FKZ: 01IS18039A","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"}],"funders":[{"id":"https://openalex.org/F4320317967","display_name":"International Max Planck Research School for Environmental, Cellular and Molecular Microbiology","ror":null},{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320321114","display_name":"Bundesministerium f\u00fcr Bildung und Forschung","ror":"https://ror.org/04pz7b180"},{"id":"https://openalex.org/F4320327553","display_name":"International Max Planck Research School for Advanced Methods in Process and Systems Engineering","ror":"https://ror.org/04hazks31"},{"id":"https://openalex.org/F4320338335","display_name":"H2020 European Research Council","ror":"https://ror.org/0472cxd90"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387995335.pdf","grobid_xml":"https://content.openalex.org/works/W4387995335.grobid-xml"},"referenced_works_count":2,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W6960139023"],"related_works":["https://openalex.org/W7101760891","https://openalex.org/W3034776059","https://openalex.org/W6903630142","https://openalex.org/W2739879705","https://openalex.org/W7078355023","https://openalex.org/W4226058880","https://openalex.org/W4226053218","https://openalex.org/W7106725816","https://openalex.org/W4415164193","https://openalex.org/W7134846041"],"abstract_inverted_index":{"Training":[0],"deep":[1],"networks":[2,27],"requires":[3],"various":[4],"design":[5],"decisions":[6],"regarding":[7],"for":[8,52,206],"instance":[9],"their":[10],"architecture,":[11],"data":[12,62,92,204],"augmentation,":[13],"or":[14,128],"optimization.":[15],"In":[16],"this":[17],"work,":[18],"we":[19,49,94,185,215,223],"find":[20],"these":[21],"training":[22],"variations":[23],"to":[24,91,100,108,138],"result":[25],"in":[26,65,125,135],"learning":[28],"unique":[29],"feature":[30],"sets":[31],"from":[32,105,149,162],"the":[33,66,175,188,226],"data.":[34],"Using":[35],"public":[36],"model":[37,59,107,140,151,156,232],"libraries":[38],"comprising":[39],"thousands":[40],"of":[41,55,70,77,177,190,230],"models":[42,79],"trained":[43],"on":[44,155,174,234],"canonical":[45],"datasets":[46],"like":[47],"ImageNet,":[48],"observe":[50],"that":[51],"arbitrary":[53,75],"pairings":[54,141],"pretrained":[56,78,139,212],"models,":[57,213],"one":[58,106],"extracts":[60],"significant":[61],"context":[63],"unavailable":[64],"other":[67],"--":[68,113],"independent":[69],"overall":[71],"performance.":[72],"Given":[73],"any":[74,150],"pairing":[76],"and":[80,146,157,195,228],"no":[81],"external":[82],"rankings":[83],"(such":[84],"as":[85,119],"separate":[86],"test":[87],"sets,":[88],"e.g.":[89],"due":[90],"privacy),":[93],"investigate":[95],"if":[96],"it":[97],"is":[98],"possible":[99],"transfer":[101,134,208],"such":[102,178],"\"complementary\"":[103],"knowledge":[104,121,147,180,192,237],"another":[109],"without":[110,153],"performance":[111],"degradation":[112],"a":[114,198],"task":[115],"made":[116],"particularly":[117],"difficult":[118],"additional":[120],"can":[122,217],"be":[123,219],"contained":[124],"stronger,":[126],"equiperformant":[127],"weaker":[129],"models.":[130,165],"Yet":[131],"facilitating":[132],"robust":[133],"scenarios":[136],"agnostic":[137],"would":[142],"unlock":[143],"auxiliary":[144],"gains":[145],"fusion":[148],"repository":[152],"restrictions":[154],"problem":[158],"specifics":[159],"-":[160],"including":[161],"weaker,":[163],"lower-performance":[164],"This":[166],"work":[167],"therefore":[168],"provides":[169],"an":[170],"initial,":[171],"in-depth":[172],"exploration":[173],"viability":[176],"general-purpose":[179],"transfer.":[181,238],"Across":[182],"large-scale":[183],"experiments,":[184],"first":[186],"reveal":[187],"shortcomings":[189],"standard":[191],"distillation":[193],"techniques,":[194],"then":[196],"propose":[197],"much":[199],"more":[200],"general":[201],"extension":[202],"through":[203],"partitioning":[205],"successful":[207,235],"between":[209],"nearly":[210],"all":[211],"which":[214],"show":[216],"also":[218],"done":[220],"unsupervised.":[221],"Finally,":[222],"assess":[224],"both":[225],"scalability":[227],"impact":[229],"fundamental":[231],"properties":[233],"model-agnostic":[236]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-10-28T00:00:00"}