{"id":"https://openalex.org/W4416176259","doi":"https://doi.org/10.48550/arxiv.2508.06601","title":"Deep Ignorance: Filtering Pretraining Data Builds Tamper-Resistant Safeguards into Open-Weight LLMs","display_name":"Deep Ignorance: Filtering Pretraining Data Builds Tamper-Resistant Safeguards into Open-Weight LLMs","publication_year":2025,"publication_date":"2025-08-08","ids":{"openalex":"https://openalex.org/W4416176259","doi":"https://doi.org/10.48550/arxiv.2508.06601"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2508.06601","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2508.06601","pdf_url":"https://arxiv.org/pdf/2508.06601","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2508.06601","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057239077","display_name":"Kyle H. O\u2019Brien","orcid":"https://orcid.org/0000-0001-5217-238X","observed_orcids":["https://orcid.org/0000-0001-5217-238X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"O'Brien, Kyle","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Casper, Stephen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Casper, Stephen","raw_affiliation_strings":["UK AI Security Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UK AI Security Institute","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015153122","display_name":"Quentin G. Anthony","orcid":"https://orcid.org/0000-0002-6823-9080","observed_orcids":["https://orcid.org/0000-0002-6823-9080"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Anthony, Quentin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114338125","display_name":"Tomek Korbak","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Korbak, Tomek","raw_affiliation_strings":["UK AI Security Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UK AI Security Institute","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Kirk, Robert","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kirk, Robert","raw_affiliation_strings":["UK AI Security Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UK AI Security Institute","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5145865735","display_name":"Xander Davies","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Davies, Xander","raw_affiliation_strings":["OATML , University of Oxford","UK AI Security Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"OATML , University of Oxford","institution_ids":["https://openalex.org/I40120149"]},{"raw_affiliation_string":"UK AI Security Institute","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101558989","display_name":"Ishan Mishra","orcid":"https://orcid.org/0009-0007-5275-5367","observed_orcids":["https://orcid.org/0009-0007-5275-5367"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mishra, Ishan","raw_affiliation_strings":["UK AI Security Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UK AI Security Institute","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029611587","display_name":"Geoffrey Irving","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Irving, Geoffrey","raw_affiliation_strings":["UK AI Security Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UK AI Security Institute","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029186201","display_name":"Yarin Gal","orcid":"https://orcid.org/0000-0002-2733-2078","observed_orcids":["https://orcid.org/0000-0002-2733-2078"]},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Gal, Yarin","raw_affiliation_strings":["OATML , University of Oxford","UK AI Security Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"OATML , University of Oxford","institution_ids":["https://openalex.org/I40120149"]},{"raw_affiliation_string":"UK AI Security Institute","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076733732","display_name":"Stella Biderman","orcid":"https://orcid.org/0000-0001-8228-1042","observed_orcids":["https://orcid.org/0000-0001-8228-1042"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Biderman, Stella","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":49,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/open-weight-large-language-models","display_name":"open-weight large language models","score":0.8930000066757202},{"id":"https://openalex.org/keywords/tamper-resistance","display_name":"tamper resistance","score":0.7229999899864197},{"id":"https://openalex.org/keywords/adversarial-fine-tuning","display_name":"adversarial fine-tuning","score":0.8050000071525574},{"id":"https://openalex.org/keywords/model-risk-management","display_name":"model risk management","score":0.6430000066757202},{"id":"https://openalex.org/keywords/defense-in-depth","display_name":"defense-in-depth","score":0.7609999775886536},{"id":"https://openalex.org/keywords/training-data-curation","display_name":"training data curation","score":0.5889999866485596}],"concepts":[{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.7569000124931335},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7067000269889832},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6703000068664551},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5285999774932861},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.47350001335144043},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.462799996137619},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.4334999918937683},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4072999954223633},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39910000562667847},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.39629998803138733},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.34119999408721924},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3287999927997589},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.3240000009536743},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2838999927043915},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2703999876976013},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.27000001072883606},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2678999900817871},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.26739999651908875},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.2587999999523163}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2508.06601","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2508.06601","pdf_url":"https://arxiv.org/pdf/2508.06601","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2508.06601","doi":"https://doi.org/10.48550/arxiv.2508.06601","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2508.06601","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2508.06601","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2508.06601","pdf_url":"https://arxiv.org/pdf/2508.06601","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416176259.pdf","grobid_xml":"https://content.openalex.org/works/W4416176259.grobid-xml"},"referenced_works_count":89,"referenced_works":["https://openalex.org/W2525332836","https://openalex.org/W3172314079","https://openalex.org/W3195247492","https://openalex.org/W3199369541","https://openalex.org/W3204096139","https://openalex.org/W4313304293","https://openalex.org/W4382140993","https://openalex.org/W4385374425","https://openalex.org/W4385572399","https://openalex.org/W4387355345","https://openalex.org/W4387389584","https://openalex.org/W4387390310","https://openalex.org/W4387436590","https://openalex.org/W4387508266","https://openalex.org/W4387681289","https://openalex.org/W4387928973","https://openalex.org/W4388184973","https://openalex.org/W4388787608","https://openalex.org/W4389421971","https://openalex.org/W4389500718","https://openalex.org/W4389984066","https://openalex.org/W4390215206","https://openalex.org/W4390962532","https://openalex.org/W4391632220","https://openalex.org/W4391709278","https://openalex.org/W4391835835","https://openalex.org/W4392425370","https://openalex.org/W4392678154","https://openalex.org/W4392822106","https://openalex.org/W4392933274","https://openalex.org/W4396944934","https://openalex.org/W4398879428","https://openalex.org/W4399251789","https://openalex.org/W4399597480","https://openalex.org/W4399912709","https://openalex.org/W4400267089","https://openalex.org/W4400373547","https://openalex.org/W4400376687","https://openalex.org/W4400434624","https://openalex.org/W4401042550","https://openalex.org/W4401306886","https://openalex.org/W4402669924","https://openalex.org/W4403443689","https://openalex.org/W4403578844","https://openalex.org/W4403809960","https://openalex.org/W4403814983","https://openalex.org/W4403884297","https://openalex.org/W4403928578","https://openalex.org/W4404340969","https://openalex.org/W4404354527","https://openalex.org/W4404406753","https://openalex.org/W4405030231","https://openalex.org/W4405254932","https://openalex.org/W4405621156","https://openalex.org/W4406272124","https://openalex.org/W4406975315","https://openalex.org/W4406975504","https://openalex.org/W4407184386","https://openalex.org/W4407185799","https://openalex.org/W4407368900","https://openalex.org/W4407375946","https://openalex.org/W4410202262","https://openalex.org/W4411270140","https://openalex.org/W4414827740","https://openalex.org/W4414836778","https://openalex.org/W4414891753","https://openalex.org/W4414891791","https://openalex.org/W4414939851","https://openalex.org/W4415022872","https://openalex.org/W4415306780","https://openalex.org/W4415329386","https://openalex.org/W4415343536","https://openalex.org/W4415795355","https://openalex.org/W4415797054","https://openalex.org/W4415799193","https://openalex.org/W4416035744","https://openalex.org/W4416095774","https://openalex.org/W4416139154","https://openalex.org/W4416524614","https://openalex.org/W4417078371","https://openalex.org/W4417098389","https://openalex.org/W4417266690","https://openalex.org/W4417357160","https://openalex.org/W6903852019","https://openalex.org/W6910571152","https://openalex.org/W6941243218","https://openalex.org/W6947991961","https://openalex.org/W6948593667","https://openalex.org/W7134626658"],"related_works":["https://openalex.org/W4401307749","https://openalex.org/W7162484867","https://openalex.org/W4417063440","https://openalex.org/W7212990102","https://openalex.org/W7163678819","https://openalex.org/W7166886670","https://openalex.org/W4405254932","https://openalex.org/W7129296852","https://openalex.org/W4416019575","https://openalex.org/W7165622098"],"abstract_inverted_index":{"Open-weight":[0],"AI":[1,220],"systems":[2],"offer":[3],"unique":[4],"benefits,":[5],"including":[6],"enhanced":[7],"transparency,":[8],"open":[9],"research,":[10],"and":[11,49,87,103,110,127,143],"decentralized":[12],"access.":[13],"However,":[14,168],"they":[15,130,179],"are":[16],"vulnerable":[17],"to":[18,55,59,134,140,165,207],"tampering":[19],"attacks":[20,137],"which":[21],"can":[22,83,180],"efficiently":[23],"elicit":[24],"harmful":[25],"behaviors":[26],"by":[27,154],"modifying":[28],"weights":[29],"or":[30],"activations.":[31],"Currently,":[32],"there":[33],"is":[34,187],"not":[35],"yet":[36],"a":[37,62,90,96,108,197,200,213],"robust":[38],"science":[39],"of":[40,66,146,158,216],"open-weight":[41,219],"model":[42],"risk":[43],"management.":[44],"Existing":[45],"safety":[46],"fine-tuning":[47,136],"methods":[48],"other":[50],"post-training":[51,152],"techniques":[52],"have":[53],"struggled":[54],"make":[56],"LLMs":[57],"resistant":[58],"more":[60,91],"than":[61],"few":[63],"dozen":[64],"steps":[65,142],"adversarial":[67,135],"fine-tuning.":[68],"In":[69],"this":[70],"paper,":[71],"we":[72,176],"investigate":[73],"whether":[74],"filtering":[75,102],"text":[76,148],"about":[77],"dual-use":[78],"topics":[79],"from":[80,125],"training":[81],"data":[82,101,210],"prevent":[84],"unwanted":[85],"capabilities":[86],"serve":[88],"as":[89,212],"tamper-resistant":[92],"safeguard.":[93],"We":[94,120],"introduce":[95],"multi-stage":[97],"pipeline":[98],"for":[99,113,199,218],"scalable":[100],"show":[104],"that":[105,129,178],"it":[106,186],"offers":[107],"tractable":[109],"effective":[111],"method":[112],"minimizing":[114],"biothreat":[115],"proxy":[116],"knowledge":[117],"in":[118,189],"LLMs.":[119],"pretrain":[121],"multiple":[122],"6.9B-parameter":[123],"models":[124,171],"scratch":[126],"find":[128,177],"exhibit":[131],"substantial":[132],"resistance":[133],"on":[138],"up":[139],"10,000":[141],"300M":[144],"tokens":[145],"biothreat-related":[147],"--":[149,160],"outperforming":[150],"existing":[151],"baselines":[153],"over":[155],"an":[156],"order":[157],"magnitude":[159],"with":[161],"no":[162],"observed":[163],"degradation":[164],"unrelated":[166],"capabilities.":[167],"while":[169],"filtered":[170],"lack":[172],"internalized":[173],"dangerous":[174],"knowledge,":[175],"still":[181],"leverage":[182],"such":[183],"information":[184],"when":[185],"provided":[188],"context":[190],"(e.g.,":[191],"via":[192],"search":[193],"tool":[194],"augmentation),":[195],"demonstrating":[196],"need":[198],"defense-in-depth":[201],"approach.":[202],"Overall,":[203],"these":[204],"findings":[205],"help":[206],"establish":[208],"pretraining":[209],"curation":[211],"promising":[214],"layer":[215],"defense":[217],"systems.":[221]},"counts_by_year":[{"year":2026,"cited_by_count":36},{"year":2025,"cited_by_count":13}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-10T00:00:00"}