{"id":"https://openalex.org/W4281483293","doi":"https://doi.org/10.48550/arxiv.2205.11433","title":"Informed Pre-Training on Prior Knowledge","display_name":"Informed Pre-Training on Prior Knowledge","publication_year":2022,"publication_date":"2022-05-23","ids":{"openalex":"https://openalex.org/W4281483293","doi":"https://doi.org/10.48550/arxiv.2205.11433"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2205.11433","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.11433","pdf_url":"https://arxiv.org/pdf/2205.11433","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2205.11433","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063338076","display_name":"Laura von Rueden","orcid":"https://orcid.org/0000-0002-7186-9753","observed_orcids":["https://orcid.org/0000-0002-7186-9753"]},"institutions":[{"id":"https://openalex.org/I135140700","display_name":"University of Bonn","ror":"https://ror.org/041nas322","country_code":"DE","type":"education","lineage":["https://openalex.org/I135140700"]},{"id":"https://openalex.org/I4210144576","display_name":"Fraunhofer Institute for Intelligent Analysis and Information Systems","ror":"https://ror.org/04nc32781","country_code":"DE","type":"facility","lineage":["https://openalex.org/I4210144576","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"von Rueden, Laura","raw_affiliation_strings":["Fraunhofer IAIS , University of Bonn Sankt Augustin , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer IAIS , University of Bonn Sankt Augustin , Germany","institution_ids":["https://openalex.org/I135140700","https://openalex.org/I4210144576"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020225503","display_name":"Sebastian Houben","orcid":"https://orcid.org/0000-0002-2036-419X","observed_orcids":["https://orcid.org/0000-0002-2036-419X"]},"institutions":[{"id":"https://openalex.org/I135140700","display_name":"University of Bonn","ror":"https://ror.org/041nas322","country_code":"DE","type":"education","lineage":["https://openalex.org/I135140700"]},{"id":"https://openalex.org/I4210144576","display_name":"Fraunhofer Institute for Intelligent Analysis and Information Systems","ror":"https://ror.org/04nc32781","country_code":"DE","type":"facility","lineage":["https://openalex.org/I4210144576","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Houben, Sebastian","raw_affiliation_strings":["Fraunhofer IAIS , University of Bonn Sankt Augustin , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer IAIS , University of Bonn Sankt Augustin , Germany","institution_ids":["https://openalex.org/I135140700","https://openalex.org/I4210144576"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059595574","display_name":"Kostadin Cvejoski","orcid":"https://orcid.org/0009-0003-6976-3997","observed_orcids":["https://orcid.org/0009-0003-6976-3997"]},"institutions":[{"id":"https://openalex.org/I135140700","display_name":"University of Bonn","ror":"https://ror.org/041nas322","country_code":"DE","type":"education","lineage":["https://openalex.org/I135140700"]},{"id":"https://openalex.org/I4210144576","display_name":"Fraunhofer Institute for Intelligent Analysis and Information Systems","ror":"https://ror.org/04nc32781","country_code":"DE","type":"facility","lineage":["https://openalex.org/I4210144576","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Cvejoski, Kostadin","raw_affiliation_strings":["Fraunhofer IAIS , University of Bonn Sankt Augustin , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer IAIS , University of Bonn Sankt Augustin , Germany","institution_ids":["https://openalex.org/I135140700","https://openalex.org/I4210144576"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003875445","display_name":"Christian Bauckhage","orcid":"https://orcid.org/0000-0001-6615-2128","observed_orcids":["https://orcid.org/0000-0001-6615-2128"]},"institutions":[{"id":"https://openalex.org/I135140700","display_name":"University of Bonn","ror":"https://ror.org/041nas322","country_code":"DE","type":"education","lineage":["https://openalex.org/I135140700"]},{"id":"https://openalex.org/I4210144576","display_name":"Fraunhofer Institute for Intelligent Analysis and Information Systems","ror":"https://ror.org/04nc32781","country_code":"DE","type":"facility","lineage":["https://openalex.org/I4210144576","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Bauckhage, Christian","raw_affiliation_strings":["Fraunhofer IAIS , University of Bonn Sankt Augustin , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer IAIS , University of Bonn Sankt Augustin , Germany","institution_ids":["https://openalex.org/I135140700","https://openalex.org/I4210144576"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029876355","display_name":"Nico Piatkowski","orcid":"https://orcid.org/0000-0002-6334-8042","observed_orcids":["https://orcid.org/0000-0002-6334-8042"]},"institutions":[{"id":"https://openalex.org/I135140700","display_name":"University of Bonn","ror":"https://ror.org/041nas322","country_code":"DE","type":"education","lineage":["https://openalex.org/I135140700"]},{"id":"https://openalex.org/I4210144576","display_name":"Fraunhofer Institute for Intelligent Analysis and Information Systems","ror":"https://ror.org/04nc32781","country_code":"DE","type":"facility","lineage":["https://openalex.org/I4210144576","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Piatkowski, Nico","raw_affiliation_strings":["Fraunhofer IAIS , University of Bonn Sankt Augustin , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer IAIS , University of Bonn Sankt Augustin , Germany","institution_ids":["https://openalex.org/I135140700","https://openalex.org/I4210144576"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.0038999998942017555,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.0017000000225380063,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/prior-knowledge","display_name":"prior knowledge","score":0.8849999904632568},{"id":"https://openalex.org/keywords/knowledge-graphs","display_name":"knowledge graphs","score":0.4729999899864197},{"id":"https://openalex.org/keywords/data-scarcity","display_name":"data scarcity","score":0.5860000252723694},{"id":"https://openalex.org/keywords/transfer-learning","display_name":"transfer learning","score":0.6190000176429749},{"id":"https://openalex.org/keywords/model-robustness","display_name":"model robustness","score":0.7689999938011169},{"id":"https://openalex.org/keywords/generalization","display_name":"generalization","score":0.878000020980835},{"id":"https://openalex.org/keywords/neural-network-initialization","display_name":"neural network initialization","score":0.777999997138977}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7172743678092957},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.6553030610084534},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6283470392227173},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.626973032951355},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.6219136714935303},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5865249037742615},{"id":"https://openalex.org/C2776960227","wikidata":"https://www.wikidata.org/wiki/Q2586354","display_name":"Knowledge transfer","level":2,"score":0.5254808068275452},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4882826805114746},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.44032353162765503},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.42569172382354736},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.2923237681388855},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08493146300315857},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2205.11433","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.11433","pdf_url":"https://arxiv.org/pdf/2205.11433","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:publica.fraunhofer.de:publica/461946","doi":null,"is_oa":false,"landing_page_url":"https://publica.fraunhofer.de/handle/publica/461946","pdf_url":null,"source":{"id":"https://openalex.org/S4306400318","display_name":"Fraunhofer-Publica (Fraunhofer-Gesellschaft)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4923324","host_organization_name":"Fraunhofer-Gesellschaft","host_organization_lineage":["https://openalex.org/I4923324"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"paper"},{"id":"doi:10.48550/arxiv.2205.11433","doi":"https://doi.org/10.48550/arxiv.2205.11433","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2205.11433","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2205.11433","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.11433","pdf_url":"https://arxiv.org/pdf/2205.11433","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W6966802470","https://openalex.org/W4309701352","https://openalex.org/W4403444228","https://openalex.org/W3188635098","https://openalex.org/W7133243301","https://openalex.org/W4416529023","https://openalex.org/W4387995335","https://openalex.org/W3205223424","https://openalex.org/W4385484513","https://openalex.org/W4291142600"],"abstract_inverted_index":{"When":[0],"training":[1,109],"data":[2,33,79,110],"is":[3,18,111,153],"scarce,":[4],"the":[5,13,96,104,122,128],"incorporation":[6],"of":[7,40,81,121],"additional":[8,163],"prior":[9,62],"knowledge":[10,41,65,82,91],"can":[11,147],"assist":[12],"learning":[14,55,97],"process.":[15],"While":[16],"it":[17],"common":[19],"to":[20,59,167],"initialize":[21],"neural":[22],"networks":[23],"with":[24],"weights":[25],"that":[26,86,131,137,144,159],"have":[27],"been":[28,44],"pre-trained":[29],"on":[30,36,61,89],"other":[31],"large":[32],"sets,":[34],"pre-training":[35,88,146,161],"more":[37],"concise":[38],"forms":[39],"has":[42,162],"rather":[43],"overlooked.":[45],"In":[46],"this":[47],"paper,":[48],"we":[49],"propose":[50],"a":[51,75,154],"novel":[52,155],"informed":[53,87,145],"machine":[54],"approach":[56],"and":[57,77,113,164],"suggest":[58],"pre-train":[60],"knowledge.":[63,151],"Formal":[64],"representations,":[66],"e.g.":[67],"graphs":[68],"or":[69],"equations,":[70],"are":[71,124],"first":[72],"transformed":[73],"into":[74],"small":[76],"condensed":[78],"set":[80],"prototypes.":[83],"We":[84],"show":[85],"such":[90],"prototypes":[92,129],"(i)":[93],"speeds":[94],"up":[95],"processes,":[98],"(ii)":[99],"improves":[100],"generalization":[101],"capabilities":[102],"in":[103],"regime":[105],"where":[106],"not":[107],"enough":[108],"available,":[112],"(iii)":[114],"increases":[115],"model":[116,123],"robustness.":[117],"Analyzing":[118],"which":[119,157],"parts":[120],"affected":[125],"most":[126],"by":[127],"reveals":[130],"improvements":[132],"come":[133],"from":[134],"deeper":[135],"layers":[136],"typically":[138],"represent":[139],"high-level":[140],"features.":[141],"This":[142,152],"confirms":[143],"indeed":[148],"transfer":[149],"semantic":[150],"effect,":[156],"shows":[158],"knowledge-based":[160],"complementary":[165],"strengths":[166],"existing":[168],"approaches.":[169]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1}],"updated_date":"2026-10-05T09:50:58.205359","created_date":"2022-05-26T00:00:00"}