{"id":"https://openalex.org/W4403748119","doi":"https://doi.org/10.48550/arxiv.2409.12914","title":"Evaluating Defences against Unsafe Feedback in RLHF","display_name":"Evaluating Defences against Unsafe Feedback in RLHF","publication_year":2024,"publication_date":"2024-09-19","ids":{"openalex":"https://openalex.org/W4403748119","doi":"https://doi.org/10.48550/arxiv.2409.12914"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2409.12914","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.12914","pdf_url":"https://arxiv.org/pdf/2409.12914","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2409.12914","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055033183","display_name":"Domenic Rosati","orcid":"https://orcid.org/0000-0003-2666-7615","observed_orcids":["https://orcid.org/0000-0003-2666-7615"]},"institutions":[{"id":"https://openalex.org/I129902397","display_name":"Dalhousie University","ror":"https://ror.org/01e6qks80","country_code":"CA","type":"education","lineage":["https://openalex.org/I129902397"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Rosati, Domenic","raw_affiliation_strings":["Dalhousie University","Vector Institute for Artificial Intelligence"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dalhousie University","institution_ids":["https://openalex.org/I129902397"]},{"raw_affiliation_string":"Vector Institute for Artificial Intelligence","institution_ids":["https://openalex.org/I4210127509"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Edkins, Giles","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Edkins, Giles","raw_affiliation_strings":["Independent"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Independent","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Raj, Harsh","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Raj, Harsh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113228327","display_name":"David Atanasov","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Atanasov, David","raw_affiliation_strings":["University of Toronto"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto","institution_ids":["https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042095646","display_name":"Subhabrata Majumdar","orcid":"https://orcid.org/0000-0003-3529-7820","observed_orcids":["https://orcid.org/0000-0003-3529-7820"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Majumdar, Subhabrata","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114056105","display_name":"Janarthanan Rajendran","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I129902397","display_name":"Dalhousie University","ror":"https://ror.org/01e6qks80","country_code":"CA","type":"education","lineage":["https://openalex.org/I129902397"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Rajendran, Janarthanan","raw_affiliation_strings":["Dalhousie University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dalhousie University","institution_ids":["https://openalex.org/I129902397"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056256317","display_name":"Frank Rudzicz","orcid":"https://orcid.org/0000-0002-1139-3423","observed_orcids":["https://orcid.org/0000-0002-1139-3423","https://orcid.org/0000-0002-9902-0583"]},"institutions":[{"id":"https://openalex.org/I129902397","display_name":"Dalhousie University","ror":"https://ror.org/01e6qks80","country_code":"CA","type":"education","lineage":["https://openalex.org/I129902397"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Rudzicz, Frank","raw_affiliation_strings":["Dalhousie University","Vector Institute for Artificial Intelligence"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dalhousie University","institution_ids":["https://openalex.org/I129902397"]},{"raw_affiliation_string":"Vector Institute for Artificial Intelligence","institution_ids":["https://openalex.org/I4210127509"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5093939627","display_name":"Hassan Sajjad","orcid":"https://orcid.org/0000-0002-8584-6595","observed_orcids":["https://orcid.org/0000-0002-8584-6595"]},"institutions":[{"id":"https://openalex.org/I129902397","display_name":"Dalhousie University","ror":"https://ror.org/01e6qks80","country_code":"CA","type":"education","lineage":["https://openalex.org/I129902397"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Sajjad, Hassan","raw_affiliation_strings":["Dalhousie University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dalhousie University","institution_ids":["https://openalex.org/I129902397"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9092000126838684,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9092000126838684,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.04879999905824661,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.009600000455975533,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning-from-human-feedback","display_name":"reinforcement learning from human feedback","score":0.9039999842643738},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6389999985694885},{"id":"https://openalex.org/keywords/safety-alignment","display_name":"safety alignment","score":0.6159999966621399},{"id":"https://openalex.org/keywords/reward-hacking","display_name":"reward hacking","score":0.7559999823570251},{"id":"https://openalex.org/keywords/constrained-markov-decision-process","display_name":"constrained Markov decision process","score":0.8500000238418579},{"id":"https://openalex.org/keywords/safety-guards","display_name":"safety guards","score":0.503000020980835}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.6080875396728516},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.4383406937122345},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4353695213794708},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.3297230005264282},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.20668694376945496},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.10012710094451904}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2409.12914","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.12914","pdf_url":"https://arxiv.org/pdf/2409.12914","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2409.12914","doi":"https://doi.org/10.48550/arxiv.2409.12914","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2409.12914","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2409.12914","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.12914","pdf_url":"https://arxiv.org/pdf/2409.12914","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G5992481797","display_name":"Fine-grained Interpretation of Deep Neural Network Models of NLP","funder_award_id":"rgpin-2022-03943","funder_id":"https://openalex.org/F4320334593","funder_display_name":"Natural Sciences and Engineering Research Council of Canada"}],"funders":[{"id":"https://openalex.org/F4320322337","display_name":"Killam Trusts","ror":"https://ror.org/021cbtn85"},{"id":"https://openalex.org/F4320330759","display_name":"Research Nova Scotia","ror":"https://ror.org/00x5ftp92"},{"id":"https://openalex.org/F4320331257","display_name":"Alliance de recherche num\u00e9rique du Canada","ror":"https://ror.org/010r6td27"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403748119.pdf","grobid_xml":"https://content.openalex.org/works/W4403748119.grobid-xml"},"referenced_works_count":16,"referenced_works":["https://openalex.org/W2736601468","https://openalex.org/W4223908421","https://openalex.org/W4384918448","https://openalex.org/W4385436553","https://openalex.org/W4387390310","https://openalex.org/W4388184973","https://openalex.org/W4389072658","https://openalex.org/W4392539041","https://openalex.org/W4399553916","https://openalex.org/W4401307749","https://openalex.org/W4402402057","https://openalex.org/W4402670256","https://openalex.org/W4404781940","https://openalex.org/W4415796496","https://openalex.org/W4415798700","https://openalex.org/W7128378279"],"related_works":["https://openalex.org/W4387838992","https://openalex.org/W4414449411","https://openalex.org/W4407719102","https://openalex.org/W4416782914","https://openalex.org/W4417256381","https://openalex.org/W4415795891","https://openalex.org/W4408507277","https://openalex.org/W7128512870","https://openalex.org/W4416887871","https://openalex.org/W4416014056"],"abstract_inverted_index":{"While":[0,35],"there":[1],"has":[2,38,53],"been":[3,39,55],"progress":[4],"towards":[5],"aligning":[6],"Large":[7],"Language":[8],"Models":[9],"(LLMs)":[10],"with":[11,50,187],"human":[12],"values":[13],"and":[14,32,114,150,215],"ensuring":[15],"safe":[16,91],"behaviour":[17],"at":[18],"inference":[19],"time,":[20],"safety":[21,120,125],"guards":[22,126],"can":[23],"easily":[24,105],"be":[25],"removed":[26],"when":[27],"fine":[28],"tuned":[29],"on":[30],"unsafe":[31,48,86,107,134],"harmful":[33,112,152],"datasets.":[34],"this":[36,71,141],"setting":[37,167],"treated":[40],"extensively,":[41],"another":[42],"popular":[43],"training":[44],"paradigm,":[45],"learning":[46,78,132,162],"from":[47,133,207],"feedback":[49,66,81],"reinforcement":[51],"learning,":[52],"previously":[54],"unexplored.":[56],"This":[57],"is":[58,82,172],"concerning":[59],"due":[60],"to":[61,97,130,138,155,176],"the":[62,177,185,188,208],"widespread":[63],"deployment":[64],"of":[65,77,147,210],"collection":[67],"systems.":[68],"We":[69,100,183],"address":[70],"gap":[72],"by":[73,194],"providing":[74],"an":[75,165],"analysis":[76],"settings":[79],"where":[80],"harmful,":[83],"i.e.":[84],"that":[85,102,118,123,169,190],"samples":[87],"are":[88,127,159],"preferred":[89],"over":[90],"ones":[92],"despite":[93],"model":[94],"developers":[95],"goal":[96],"maintain":[98],"safety.":[99],"find":[101],"safety-aligned":[103],"LLMs":[104],"explore":[106],"action":[108],"spaces":[109],"via":[110],"generating":[111],"text":[113],"optimize":[115],"for":[116,179,199,219],"reward":[117,197],"violates":[119],"constraints":[121,163],"indicating":[122],"current":[124],"not":[128],"enough":[129],"prevent":[131],"feedback.":[135],"In":[136],"order":[137],"protect":[139],"against":[140],"vulnerability,":[142],"we":[143,201],"adapt":[144],"a":[145,203],"number":[146],"both":[148],"\"implict\"":[149],"\"explicit\"":[151],"fine-tuning":[153],"defences":[154,192],"evaluate":[156],"whether":[157],"they":[158],"effective":[160,174],"as":[161],"in":[164],"RLHF":[166],"finding":[168],"no":[170],"method":[171],"generally":[173],"pointing":[175],"need":[178],"more":[180],"defence":[181,221],"research.":[182],"end":[184],"paper":[186],"observation":[189],"some":[191,217],"work":[193],"performing":[195],"\"harmless":[196],"hacking\"":[198],"which":[200],"provide":[202,216],"theoretical":[204],"explanation":[205],"drawn":[206],"theory":[209],"Constrained":[211],"Markov":[212],"Decision":[213],"Processes":[214],"direction":[218],"future":[220],"development.":[222]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-10-25T00:00:00"}